From a888249770cec8be42d33b51ef455826851688ee Mon Sep 17 00:00:00 2001
From: Adrian Holovaty <adrian@holovaty.com>
Date: Tue, 7 Nov 2006 02:32:33 +0000
Subject: [PATCH] Implemented [4035] (improvement in URL validation) in
 django.newforms

git-svn-id: http://code.djangoproject.com/svn/django/trunk@4037 bcc190cf-cafb-0310-a4f2-bffc1f526a37
---
 django/newforms/fields.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/django/newforms/fields.py b/django/newforms/fields.py
index 79558bbf9e19d..54089cb3c3241 100644
--- a/django/newforms/fields.py
+++ b/django/newforms/fields.py
@@ -188,17 +188,35 @@ def __init__(self, required=True, widget=None):
     r'(?::\d+)?' # optional port
     r'(?:/?|/\S+)$', re.IGNORECASE)
 
+try:
+    from django.conf import settings
+    URL_VALIDATOR_USER_AGENT = settings.URL_VALIDATOR_USER_AGENT
+except ImportError:
+    # It's OK if Django settings aren't configured.
+    URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)'
+
 class URLField(RegexField):
-    def __init__(self, required=True, verify_exists=False, widget=None):
+    def __init__(self, required=True, verify_exists=False, widget=None,
+            validator_user_agent=URL_VALIDATOR_USER_AGENT):
         RegexField.__init__(self, url_re, u'Enter a valid URL.', required, widget)
         self.verify_exists = verify_exists
+        self.user_agent = validator_user_agent
 
     def clean(self, value):
         value = RegexField.clean(self, value)
         if self.verify_exists:
             import urllib2
+            from django.conf import settings
+            headers = {
+                "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
+                "Accept-Language": "en-us,en;q=0.5",
+                "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
+                "Connection": "close",
+                "User-Agent": self.user_agent,
+            }
             try:
-                u = urllib2.urlopen(value)
+                req = urllib2.Request(field_data, None, headers)
+                u = urllib2.urlopen(req)
             except ValueError:
                 raise ValidationError(u'Enter a valid URL.')
             except: # urllib2.URLError, httplib.InvalidURL, etc.