Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Removed deprecated URLField.verify_exists.

The deprecation schedule was slightly accelerated because of possible security ramifications.



git-svn-id: http://code.djangoproject.com/svn/django/trunk@17847 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit 9ed6e08ff99c18710c0e4875f827235f04c89d76 1 parent 800e394
@aaugustin aaugustin authored
View
5 django/conf/global_settings.py
@@ -403,11 +403,6 @@
# Hint: you really don't!
TRANSACTIONS_MANAGED = False
-# The User-Agent string to use when checking for URL validity through the
-# isExistingURL validator.
-from django import get_version
-URL_VALIDATOR_USER_AGENT = "Django/%s (https://www.djangoproject.com)" % get_version()
-
# The tablespaces to use for each model when not specified otherwise.
DEFAULT_TABLESPACE = ''
DEFAULT_INDEX_TABLESPACE = ''
View
66 django/core/validators.py
@@ -1,6 +1,4 @@
import re
-import urllib
-import urllib2
import urlparse
from django.core.exceptions import ValidationError
@@ -11,13 +9,6 @@
# These values, if given to validate(), will trigger the self.required check.
EMPTY_VALUES = (None, '', [], (), {})
-try:
- from django.conf import settings
- URL_VALIDATOR_USER_AGENT = settings.URL_VALIDATOR_USER_AGENT
-except ImportError:
- # It's OK if Django settings aren't configured.
- URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)'
-
class RegexValidator(object):
regex = ''
message = _(u'Enter a valid value.')
@@ -51,11 +42,8 @@ class URLValidator(RegexValidator):
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
- def __init__(self, verify_exists=False,
- validator_user_agent=URL_VALIDATOR_USER_AGENT):
+ def __init__(self):
super(URLValidator, self).__init__()
- self.verify_exists = verify_exists
- self.user_agent = validator_user_agent
def __call__(self, value):
try:
@@ -76,58 +64,6 @@ def __call__(self, value):
else:
url = value
- if self.verify_exists:
- import warnings
- warnings.warn(
- "The URLField verify_exists argument has intractable security "
- "and performance issues. Accordingly, it has been deprecated.",
- DeprecationWarning
- )
-
- headers = {
- "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
- "Accept-Language": "en-us,en;q=0.5",
- "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
- "Connection": "close",
- "User-Agent": self.user_agent,
- }
- url = url.encode('utf-8')
- # Quote characters from the unreserved set, refs #16812
- url = urllib.quote(url, "!*'();:@&=+$,/?#[]")
- broken_error = ValidationError(
- _(u'This URL appears to be a broken link.'), code='invalid_link')
- try:
- req = urllib2.Request(url, None, headers)
- req.get_method = lambda: 'HEAD'
- #Create an opener that does not support local file access
- opener = urllib2.OpenerDirector()
-
- #Don't follow redirects, but don't treat them as errors either
- error_nop = lambda *args, **kwargs: True
- http_error_processor = urllib2.HTTPErrorProcessor()
- http_error_processor.http_error_301 = error_nop
- http_error_processor.http_error_302 = error_nop
- http_error_processor.http_error_307 = error_nop
-
- handlers = [urllib2.UnknownHandler(),
- urllib2.HTTPHandler(),
- urllib2.HTTPDefaultErrorHandler(),
- urllib2.FTPHandler(),
- http_error_processor]
- try:
- import ssl
- except ImportError:
- # Python isn't compiled with SSL support
- pass
- else:
- handlers.append(urllib2.HTTPSHandler())
- map(opener.add_handler, handlers)
- opener.open(req, timeout=10)
- except ValueError:
- raise ValidationError(_(u'Enter a valid URL.'), code='invalid')
- except: # urllib2.URLError, httplib.InvalidURL, etc.
- raise broken_error
-
def validate_integer(value):
try:
View
8 django/db/models/fields/__init__.py
@@ -890,7 +890,7 @@ class EmailField(CharField):
description = _("E-mail address")
def __init__(self, *args, **kwargs):
- # max_length should be overridden to 254 characters to be fully
+ # max_length should be overridden to 254 characters to be fully
# compliant with RFCs 3696 and 5321
kwargs['max_length'] = kwargs.get('max_length', 75)
@@ -1251,12 +1251,10 @@ def formfield(self, **kwargs):
class URLField(CharField):
description = _("URL")
- def __init__(self, verbose_name=None, name=None, verify_exists=False,
- **kwargs):
+ def __init__(self, verbose_name=None, name=None, **kwargs):
kwargs['max_length'] = kwargs.get('max_length', 200)
CharField.__init__(self, verbose_name, name, **kwargs)
- self.validators.append(
- validators.URLValidator(verify_exists=verify_exists))
+ self.validators.append(validators.URLValidator())
def formfield(self, **kwargs):
# As with CharField, this will cause URL validation to be performed
View
9 django/forms/fields.py
@@ -598,14 +598,11 @@ def to_python(self, data):
class URLField(CharField):
default_error_messages = {
'invalid': _(u'Enter a valid URL.'),
- 'invalid_link': _(u'This URL appears to be a broken link.'),
}
- def __init__(self, max_length=None, min_length=None, verify_exists=False,
- validator_user_agent=validators.URL_VALIDATOR_USER_AGENT, *args, **kwargs):
- super(URLField, self).__init__(max_length, min_length, *args,
- **kwargs)
- self.validators.append(validators.URLValidator(verify_exists=verify_exists, validator_user_agent=validator_user_agent))
+ def __init__(self, max_length=None, min_length=None, *args, **kwargs):
+ super(URLField, self).__init__(max_length, min_length, *args, **kwargs)
+ self.validators.append(validators.URLValidator())
def to_python(self, value):
View
16 docs/ref/forms/fields.txt
@@ -789,7 +789,7 @@ For each field, we describe the default widget used if you don't specify
* Empty value: ``''`` (an empty string)
* Normalizes to: A Unicode object.
* Validates that the given value is a valid URL.
- * Error message keys: ``required``, ``invalid``, ``invalid_link``
+ * Error message keys: ``required``, ``invalid``
Takes the following optional arguments:
@@ -798,20 +798,6 @@ For each field, we describe the default widget used if you don't specify
These are the same as ``CharField.max_length`` and ``CharField.min_length``.
- .. attribute:: verify_exists
-
- If ``True``, the validator will attempt to load the given URL, raising
- ``ValidationError`` if the page gives a 404. Defaults to ``False``.
-
- .. deprecated:: 1.4
- ``verify_exists`` was deprecated for security reasons and will be removed in
- Django 1.5. This deprecation also removes ``validator_user_agent``.
-
- .. attribute:: validator_user_agent
-
- String used as the user-agent used when checking for a URL's existence.
- Defaults to the value of the :setting:`URL_VALIDATOR_USER_AGENT` setting.
-
.. versionchanged:: 1.2
The URLField previously did not recognize URLs as valid that contained an IDN
(Internationalized Domain Name; a domain name containing unicode characters)
View
26 docs/ref/models/fields.txt
@@ -484,10 +484,10 @@ A :class:`CharField` that checks that the value is a valid email address.
.. admonition:: Incompliance to RFCs
The default 75 character ``max_length`` is not capable of storing all
- possible RFC3696/5321-compliant email addresses. In order to store all
- possible valid email addresses, a ``max_length`` of 254 is required.
- The default ``max_length`` of 75 exists for historical reasons. The
- default has not been changed in order to maintain backwards
+ possible RFC3696/5321-compliant email addresses. In order to store all
+ possible valid email addresses, a ``max_length`` of 254 is required.
+ The default ``max_length`` of 75 exists for historical reasons. The
+ default has not been changed in order to maintain backwards
compatibility with existing uses of :class:`EmailField`.
``FileField``
@@ -887,23 +887,9 @@ shortcuts.
``URLField``
------------
-.. class:: URLField([verify_exists=False, max_length=200, **options])
+.. class:: URLField([max_length=200, **options])
-A :class:`CharField` for a URL. Has one extra optional argument:
-
-.. deprecated:: 1.4
- ``verify_exists`` is deprecated for security reasons as of 1.4 and will be
- removed in Django 1.5. Prior to 1.3.1, the default value was ``True``.
-
-.. attribute:: URLField.verify_exists
-
- If ``True``, the URL given will be checked for existence (i.e.,
- the URL actually loads and doesn't give a 404 response) using a
- ``HEAD`` request. Redirects are allowed, but will not be followed.
-
- Note that when you're using the single-threaded development server,
- validating a URL being served by the same server will hang. This should not
- be a problem for multithreaded servers.
+A :class:`CharField` for a URL.
The admin represents this as an ``<input type="text">`` (a single-line input).
View
21 docs/ref/settings.txt
@@ -2148,18 +2148,6 @@ to ensure your processes are running in the correct environment.
.. _pytz: http://pytz.sourceforge.net/
-.. setting:: URL_VALIDATOR_USER_AGENT
-
-URL_VALIDATOR_USER_AGENT
-------------------------
-
-Default: ``Django/<version> (https://www.djangoproject.com/)``
-
-The string to use as the ``User-Agent`` header when checking to see if
-URLs exist (see the ``verify_exists`` option on
-:class:`~django.db.models.URLField`). This setting was deprecated in
-1.3.1 along with ``verify_exists`` and will be removed in 1.4.
-
.. setting:: USE_ETAGS
USE_ETAGS
@@ -2344,3 +2332,12 @@ IGNORABLE_404_STARTS
.. deprecated:: 1.4
This setting has been superseded by :setting:`IGNORABLE_404_URLS`.
+
+.. setting:: URL_VALIDATOR_USER_AGENT
+
+URL_VALIDATOR_USER_AGENT
+------------------------
+
+.. deprecated:: 1.5
+ This value was used as the ``User-Agent`` header when checking if a URL
+ exists, a feature that was removed due to security and performance issues.
View
22 docs/ref/validators.txt
@@ -88,26 +88,10 @@ to, or in lieu of custom ``field.clean()`` methods.
``URLValidator``
----------------
-.. class:: URLValidator([verify_exists=False, validator_user_agent=URL_VALIDATOR_USER_AGENT])
+.. class:: URLValidator()
- A :class:`RegexValidator` that ensures a value looks like a URL and
- optionally verifies that the URL actually exists (i.e., doesn't return a
- 404 status code). Raises an error code of ``'invalid'`` if it doesn't look
- like a URL, and a code of ``'invalid_link'`` if it doesn't exist.
-
- :param verify_exists: Sets :attr:`verify_exists`. Defaults to ``False``.
- :param validator_user_agent: Sets :attr:`validator_user_agent`. Defaults to
- :setting:`URL_VALIDATOR_USER_AGENT` or, if that setting is set to a
- null value, ``"Django (https://www.djangoproject.com/)"``.
-
- .. attribute:: verify_exists
-
- If set to ``True``, this validator checks that the URL actually exists.
-
- .. attribute:: validator_user_agent
-
- If :attr:`verify_exists` is ``True``, Django uses this value as the
- "User-agent" for the request.
+ A :class:`RegexValidator` that ensures a value looks like a URL, and raises
+ an error code of ``'invalid'`` if it doesn't.
``validate_email``
------------------
View
3  docs/topics/forms/modelforms.txt
@@ -106,8 +106,7 @@ Model field Form field
``TimeField`` ``TimeField``
-``URLField`` ``URLField`` with ``verify_exists`` set
- to the model field's ``verify_exists``
+``URLField`` ``URLField``
=============================== ========================================
.. versionadded:: 1.2
View
3  tests/modeltests/validation/models.py
@@ -15,7 +15,6 @@ class ModelToValidate(models.Model):
parent = models.ForeignKey('self', blank=True, null=True, limit_choices_to={'number': 10})
email = models.EmailField(blank=True)
url = models.URLField(blank=True)
- url_verify = models.URLField(blank=True, verify_exists=True)
f_with_custom_validator = models.IntegerField(blank=True, null=True, validators=[validate_answer_to_universe])
def clean(self):
@@ -102,4 +101,4 @@ class MultipleAutoFields(models.Model):
auto2 = models.AutoField(primary_key=True)
except AssertionError, assertion_error:
pass # Fail silently
-assert str(assertion_error) == u"A model can't have more than one AutoField."
+assert str(assertion_error) == u"A model can't have more than one AutoField."
View
32 tests/modeltests/validation/tests.py
@@ -1,14 +1,9 @@
from __future__ import absolute_import
-import warnings
-
from django import forms
from django.core.exceptions import NON_FIELD_ERRORS
from django.test import TestCase
-# Import the verify_exists_urls from the 'forms' test app
-from regressiontests.forms.tests.fields import verify_exists_urls
-
from . import ValidationTestCase
from .models import (Author, Article, ModelToValidate,
GenericIPAddressTestModel, GenericIPAddrUnpackUniqueTest)
@@ -22,14 +17,6 @@
class BaseModelValidationTests(ValidationTestCase):
- def setUp(self):
- self.save_warnings_state()
- warnings.filterwarnings('ignore', category=DeprecationWarning,
- module='django.core.validators')
-
- def tearDown(self):
- self.restore_warnings_state()
-
def test_missing_required_field_raises_error(self):
mtv = ModelToValidate(f_with_custom_validator=42)
self.assertFailsValidation(mtv.full_clean, ['name', 'number'])
@@ -70,25 +57,6 @@ def test_wrong_url_value_raises_error(self):
mtv = ModelToValidate(number=10, name='Some Name', url='not a url')
self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'Enter a valid value.'])
- #The tests below which use url_verify are deprecated
- def test_correct_url_but_nonexisting_gives_404(self):
- mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=404')
- self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.'])
-
- @verify_exists_urls(existing_urls=('http://www.google.com/',))
- def test_correct_url_value_passes(self):
- mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://www.google.com/')
- self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection
-
- @verify_exists_urls(existing_urls=('http://qa-dev.w3.org/link-testsuite/http.php?code=301',))
- def test_correct_url_with_redirect(self):
- mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=301') #example.com is a redirect to iana.org now
- self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection
-
- def test_correct_https_url_but_nonexisting(self):
- mtv = ModelToValidate(number=10, name='Some Name', url_verify='https://www.example.com/')
- self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.'])
-
def test_text_greater_that_charfields_max_length_raises_erros(self):
mtv = ModelToValidate(number=10, name='Some Name'*100)
self.assertFailsValidation(mtv.full_clean, ['name',])
View
7 tests/regressiontests/forms/tests/error_messages.py
@@ -7,8 +7,6 @@
from django.test import TestCase
from django.utils.safestring import mark_safe
-from .fields import verify_exists_urls
-
class AssertFormErrorsMixin(object):
def assertFormErrors(self, expected, the_callable, *args, **kwargs):
@@ -143,17 +141,14 @@ def test_filefield(self):
self.assertFormErrors([u'EMPTY FILE'], f.clean, SimpleUploadedFile('name', None))
self.assertFormErrors([u'EMPTY FILE'], f.clean, SimpleUploadedFile('name', ''))
- @verify_exists_urls()
def test_urlfield(self):
e = {
'required': 'REQUIRED',
'invalid': 'INVALID',
- 'invalid_link': 'INVALID LINK',
}
- f = URLField(verify_exists=True, error_messages=e)
+ f = URLField(error_messages=e)
self.assertFormErrors([u'REQUIRED'], f.clean, '')
self.assertFormErrors([u'INVALID'], f.clean, 'abc.c')
- self.assertFormErrors([u'INVALID LINK'], f.clean, 'http://www.broken.djangoproject.com')
def test_booleanfield(self):
e = {
View
61 tests/regressiontests/forms/tests/fields.py
@@ -27,10 +27,8 @@
import datetime
import re
import os
-import urllib2
import warnings
from decimal import Decimal
-from functools import wraps
from django.core.files.uploadedfile import SimpleUploadedFile
from django.forms import *
@@ -48,32 +46,6 @@ def fix_os_paths(x):
return x
-def verify_exists_urls(existing_urls=()):
- """
- Patches urllib to simulate the availability of some urls even when there
- is no Internet connection. This hack should be removed alongside with
- `URLField.verify_exists` in Django 1.5.
- """
- def decorator(func):
- @wraps(func)
- def wrapper(*args, **kwargs):
- from django.core import validators
- # patch urllib2.OpenerDirector
- original_open = validators.urllib2.OpenerDirector.open
- def custom_open(self, req, data=None, timeout=None):
- if req.get_full_url() in existing_urls:
- return True
- raise Exception()
- try:
- urllib2.OpenerDirector.open = custom_open
- func(*args, **kwargs)
- finally:
- # unpatch urllib2.OpenerDirector
- validators.urllib2.OpenerDirector.open = original_open
- return wrapper
- return decorator
-
-
class FieldsTests(SimpleTestCase):
def setUp(self):
@@ -651,30 +623,6 @@ def test_urlfield_2(self):
self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://example.')
self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://.com')
- @verify_exists_urls(('http://www.google.com/',))
- def test_urlfield_3(self):
- f = URLField(verify_exists=True)
- self.assertEqual(u'http://www.google.com/', f.clean('http://www.google.com'))
- self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://example')
- self.assertRaises(ValidationError, f.clean, 'http://www.broken.djangoproject.com') # bad domain
- self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=405') # Method not allowed
- try:
- f.clean('http://www.broken.djangoproject.com') # bad domain
- except ValidationError, e:
- self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
- self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=400') # good domain, bad page
- try:
- f.clean('http://google.com/we-love-microsoft.html') # good domain, bad page
- except ValidationError, e:
- self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
-
- @verify_exists_urls(('http://www.google.com/',))
- def test_urlfield_4(self):
- f = URLField(verify_exists=True, required=False)
- self.assertEqual(u'', f.clean(''))
- self.assertEqual(u'http://www.google.com/', f.clean('http://www.google.com'))
-
- @verify_exists_urls(('http://example.com/',))
def test_urlfield_5(self):
f = URLField(min_length=15, max_length=20)
self.assertRaisesMessage(ValidationError, "[u'Ensure this value has at least 15 characters (it has 13).']", f.clean, 'http://f.com')
@@ -698,7 +646,7 @@ def test_urlfield_8(self):
self.assertEqual(u'http://example.com/?some_param=some_value', f.clean('http://example.com?some_param=some_value'))
def test_urlfield_9(self):
- f = URLField(verify_exists=False)
+ f = URLField()
urls = (
u'http://עברית.idn.icann.org/',
u'http://sãopaulo.com/',
@@ -722,13 +670,6 @@ def test_urlfield_9(self):
except ValidationError, e:
self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
- @verify_exists_urls((u'http://xn--hxargifdar.idn.icann.org/%CE%91%CF%81%CF%87%CE%B9%CE%BA%CE%AE_%CF%83%CE%B5%CE%BB%CE%AF%CE%B4%CE%B1',))
- def test_urlfield_10(self):
- # UTF-8 in the domain.
- f = URLField(verify_exists=True)
- url = u'http://\u03b5\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac.idn.icann.org/\u0391\u03c1\u03c7\u03b9\u03ba\u03ae_\u03c3\u03b5\u03bb\u03af\u03b4\u03b1'
- self.assertEqual(url, f.clean(url))
-
def test_urlfield_not_string(self):
f = URLField(required=False)
self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 23)
View
2  tests/regressiontests/generic_inline_admin/models.py
@@ -16,7 +16,7 @@ class Media(models.Model):
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
content_object = generic.GenericForeignKey()
- url = models.URLField(verify_exists=False)
+ url = models.URLField()
description = models.CharField(max_length=100, blank=True)
keywords = models.CharField(max_length=100, blank=True)
View
2  tests/regressiontests/model_forms_regress/models.py
@@ -57,7 +57,7 @@ class Author1(models.Model):
full_name = models.CharField(max_length=255)
class Homepage(models.Model):
- url = models.URLField(verify_exists=False)
+ url = models.URLField()
class Document(models.Model):
myfile = models.FileField(upload_to='unused', blank=True)
Please sign in to comment.
Something went wrong with that request. Please try again.