Merge branch 'master' into stats-blueprint

ckan/config/environment.py

@@ -4,13 +4,14 @@
 import os
 import logging
 import warnings
-from urlparse import urlparse
 import pytz
 
 import sqlalchemy
 from pylons import config as pylons_config
 import formencode
 
+from six.moves.urllib.parse import urlparse
+
 import ckan.config.routing as routing
 import ckan.model as model
 import ckan.plugins as p

ckan/config/middleware/__init__.py

@@ -1,13 +1,12 @@
 # encoding: utf-8
 
 """WSGI app initialization"""
-import urllib
-import urlparse
-import urllib
 
 import webob
 from routes import request_config as routes_request_config
 
+from six.moves.urllib.parse import urlparse, quote
+
 from ckan.lib.i18n import get_locales_from_config
 from ckan.config.environment import load_environment
 from ckan.config.middleware.flask_app import make_flask_stack
@@ -147,13 +146,13 @@ def handle_i18n(self, environ):
             path_info = environ['PATH_INFO']
             # sort out weird encodings
             path_info = \
-                '/'.join(urllib.quote(pce, '') for pce in path_info.split('/'))
+                '/'.join(quote(pce, '') for pce in path_info.split('/'))
 
             qs = environ.get('QUERY_STRING')
 
             if qs:
                 # sort out weird encodings
-                qs = urllib.quote(qs, '')
+                qs = quote(qs, '')
                 environ['CKAN_CURRENT_URL'] = '%s?%s' % (path_info, qs)
             else:
                 environ['CKAN_CURRENT_URL'] = path_info
@@ -193,8 +192,8 @@ def __call__(self, environ, start_response):
         if app_name == 'flask_app':
             # This request will be served by Flask, but we still need the
             # Pylons URL builder (Routes) to work
-            parts = urlparse.urlparse(config.get('ckan.site_url',
-                                                 'http://0.0.0.0:5000'))
+            parts = urlparse(
+                config.get('ckan.site_url', 'http://0.0.0.0:5000'))
             request_config = routes_request_config()
             request_config.host = str(parts.netloc + parts.path)
             request_config.protocol = str(parts.scheme)

ckan/config/middleware/common_middleware.py

@@ -1,12 +1,12 @@
 # encoding: utf-8
 
 """Common middleware used by both Flask and Pylons app stacks."""
-
-import urllib2
 import hashlib
 import json
 import cgi
 
+from six.moves.urllib.parse import unquote
+
 import sqlalchemy as sa
 from webob.request import FakeCGIBody
 
@@ -73,7 +73,7 @@ def __call__(self, environ, start_response):
             data = {}
             for part in parts:
                 k, v = part.split('=')
-                data[k] = urllib2.unquote(v).decode("utf8")
+                data[k] = unquote(v).decode("utf8")
             start_response('200 OK', [('Content-Type', 'text/html')])
             # we want a unique anonomized key for each user so that we do
             # not count multiple clicks from the same user.

ckan/config/middleware/flask_app.py

@@ -82,13 +82,14 @@ def make_flask_stack(conf, **app_conf):
     debug = asbool(conf.get('debug', conf.get('DEBUG', False)))
     testing = asbool(app_conf.get('testing', app_conf.get('TESTING', False)))
     app = flask_app = CKANFlask(__name__)
+    app.jinja_options = jinja_extensions.get_jinja_env_options()
+
     app.debug = debug
     app.testing = testing
     app.template_folder = os.path.join(root, 'templates')
     app.app_ctx_globals_class = CKAN_AppCtxGlobals
     app.url_rule_class = CKAN_Rule
 
-    app.jinja_options = jinja_extensions.get_jinja_env_options()
     # Update Flask config with the CKAN values. We use the common config
     # object as values might have been modified on `load_environment`
     if config:

ckan/controllers/api.py

@@ -3,9 +3,9 @@
 import os.path
 import logging
 import cgi
-import urllib
 
 from six import text_type
+from six.moves.urllib.parse import unquote_plus
 
 import ckan.model as model
 import ckan.logic as logic
@@ -383,7 +383,7 @@ def make_unicode(entity):
                 if keys and request.POST[keys[0]] in [u'1', u'']:
                     request_data = keys[0]
                 else:
-                    request_data = urllib.unquote_plus(request.body)
+                    request_data = unquote_plus(request.body)
             except Exception as inst:
                 msg = "Could not find the POST data: %r : %s" % \
                       (request.POST, inst)

ckan/controllers/feed.py

@@ -22,9 +22,10 @@
 """
 # TODO fix imports
 import logging
-import urlparse
 
 from six import text_type
+from six.moves.urllib.parse import urlparse
+
 import webhelpers.feedgenerator
 
 import ckan.lib.base as base
@@ -131,7 +132,7 @@ def _create_atom_id(resource_path, authority_name=None, date_string=None):
         authority_name = config.get('ckan.feeds.authority_name', '').strip()
         if not authority_name:
             site_url = config.get('ckan.site_url', '').strip()
-            authority_name = urlparse.urlparse(site_url).netloc
+            authority_name = urlparse(site_url).netloc
 
     if not authority_name:
         log.warning('No authority_name available for feed generation.  '

ckan/controllers/group.py

@@ -2,7 +2,7 @@
 
 import logging
 import datetime
-from urllib import urlencode
+from six.moves.urllib.parse import urlencode
 
 from pylons.i18n import get_lang
 from six import string_types, text_type

ckan/controllers/package.py

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 import logging
-from urllib import urlencode
+from six.moves.urllib.parse import urlencode
 import datetime
 import mimetypes
 import cgi

ckan/lib/captcha.py

@@ -2,9 +2,8 @@
 
 from ckan.common import config
 
-import urllib
-import urllib2
-import json
+import requests
+
 
 def check_recaptcha(request):
     '''Check a user\'s recaptcha submission is valid, and raise CaptchaError
@@ -14,7 +13,8 @@ def check_recaptcha(request):
         # Recaptcha not enabled
         return
 
-    client_ip_address = request.environ.get('REMOTE_ADDR', 'Unknown IP Address')
+    client_ip_address = request.environ.get(
+        'REMOTE_ADDR', 'Unknown IP Address')
 
     # reCAPTCHA v2
     recaptcha_response_field = request.form.get('g-recaptcha-response', '')
@@ -23,12 +23,13 @@ def check_recaptcha(request):
     # recaptcha_response_field will be unicode if there are foreign chars in
     # the user input. So we need to encode it as utf8 before urlencoding or
     # we get an exception (#1431).
-    params = urllib.urlencode(dict(secret=recaptcha_private_key,
-                                   remoteip=client_ip_address,
-                                   response=recaptcha_response_field.encode('utf8')))
-    f = urllib2.urlopen(recaptcha_server_name, params)
-    data = json.load(f)
-    f.close()
+    params = dict(
+        secret=recaptcha_private_key,
+        remoteip=client_ip_address,
+        response=recaptcha_response_field.encode('utf8')
+    )
+    response = requests.get(recaptcha_server_name, params)
+    data = response.json()
 
     try:
         if not data['success']:
@@ -37,5 +38,6 @@ def check_recaptcha(request):
         # Something weird with recaptcha response
         raise CaptchaError()
 
+
 class CaptchaError(ValueError):
     pass

ckan/lib/datapreview.py

@@ -5,9 +5,10 @@
 Functions and data structures that are needed for the ckan data preview.
 """
 
-import urlparse
 import logging
 
+from six.moves.urllib.parse import urlparse
+
 from ckan.common import config
 
 import ckan.plugins as p
@@ -36,9 +37,9 @@ def compare_domains(urls):
         # all urls are interpreted as absolute urls,
         # except for urls that start with a /
         try:
-            if not urlparse.urlparse(url).scheme and not url.startswith('/'):
+            if not urlparse(url).scheme and not url.startswith('/'):
                 url = '//' + url
-            parsed = urlparse.urlparse(url.lower(), 'http')
+            parsed = urlparse(url.lower(), 'http')
             domain = (parsed.scheme, parsed.hostname, parsed.port)
         except ValueError:
             # URL is so messed up that even urlparse can't stand it

ckan/lib/dictization/model_dictize.py

@@ -12,7 +12,8 @@
 which builds the dictionary by iterating over the table columns.
 '''
 import datetime
-import urlparse
+
+from six.moves.urllib.parse import urlsplit
 
 from ckan.common import config
 from sqlalchemy.sql import select
@@ -115,7 +116,7 @@ def resource_dictize(res, context):
                                     resource_id=res.id,
                                     filename=cleaned_name,
                                     qualified=True)
-    elif resource['url'] and not urlparse.urlsplit(url).scheme and not context.get('for_edit'):
+    elif resource['url'] and not urlsplit(url).scheme and not context.get('for_edit'):
         resource['url'] = u'http://' + url.lstrip('/')
     return resource
 

ckan/lib/helpers.py

@@ -12,11 +12,8 @@
 import os
 import pytz
 import tzlocal
-import urllib
 import pprint
 import copy
-import urlparse
-from urllib import urlencode
 import uuid
 
 from paste.deploy import converters
@@ -35,7 +32,11 @@
 from flask import url_for as _flask_default_url_for
 from werkzeug.routing import BuildError as FlaskRouteBuildError
 import i18n
+
 from six import string_types, text_type
+from six.moves.urllib.parse import (
+    urlencode, quote, unquote, urlparse, urlunparse
+)
 import jinja2
 
 import ckan.exceptions
@@ -233,7 +234,7 @@ def get_site_protocol_and_host():
     '''
     site_url = config.get('ckan.site_url', None)
     if site_url is not None:
-        parsed_url = urlparse.urlparse(site_url)
+        parsed_url = urlparse(site_url)
         return (
             parsed_url.scheme.encode('utf-8'),
             parsed_url.netloc.encode('utf-8')
@@ -395,9 +396,9 @@ def _url_for_flask(*args, **kw):
         # Flask to pass the host explicitly, so we rebuild the URL manually
         # based on `ckan.site_url`, which is essentially what we did on Pylons
         protocol, host = get_site_protocol_and_host()
-        parts = urlparse.urlparse(my_url)
-        my_url = urlparse.urlunparse((protocol, host, parts.path, parts.params,
-                                      parts.query, parts.fragment))
+        parts = urlparse(my_url)
+        my_url = urlunparse((protocol, host, parts.path, parts.params,
+                             parts.query, parts.fragment))
 
     return my_url
 
@@ -435,7 +436,7 @@ def url_for_static(*args, **kw):
     This is a wrapper for :py:func:`routes.url_for`
     '''
     if args:
-        url = urlparse.urlparse(args[0])
+        url = urlparse(args[0])
         url_is_external = (url.scheme != '' or url.netloc != '')
         if url_is_external:
             CkanUrlException = ckan.exceptions.CkanUrlException
@@ -451,7 +452,7 @@ def url_for_static_or_external(*args, **kw):
     This is a wrapper for :py:func:`routes.url_for`
     '''
     def fix_arg(arg):
-        url = urlparse.urlparse(str(arg))
+        url = urlparse(str(arg))
         url_is_relative = (url.scheme == '' and url.netloc == '' and
                            not url.path.startswith('/'))
         if url_is_relative:
@@ -474,7 +475,7 @@ def is_url(*args, **kw):
     if not args:
         return False
     try:
-        url = urlparse.urlparse(args[0])
+        url = urlparse(args[0])
     except ValueError:
         return False
 
@@ -556,9 +557,9 @@ def url_is_local(url):
     '''Returns True if url is local'''
     if not url or url.startswith('//'):
         return False
-    parsed = urlparse.urlparse(url)
+    parsed = urlparse(url)
     if parsed.scheme:
-        domain = urlparse.urlparse(url_for('/', qualified=True)).netloc
+        domain = urlparse(url_for('/', qualified=True)).netloc
         if domain != parsed.netloc:
             return False
     return True
@@ -574,7 +575,7 @@ def full_current_url():
 @core_helper
 def current_url():
     ''' Returns current url unquoted'''
-    return urllib.unquote(request.environ['CKAN_CURRENT_URL'])
+    return unquote(request.environ['CKAN_CURRENT_URL'])
 
 
 @core_helper
@@ -1333,7 +1334,7 @@ def gravatar(email_hash, size=100, default=None):
 
     if default not in _VALID_GRAVATAR_DEFAULTS:
         # treat the default as a url
-        default = urllib.quote(default, safe='')
+        default = quote(default, safe='')
 
     return literal('''<img src="//gravatar.com/avatar/%s?s=%d&amp;d=%s"
         class="gravatar" width="%s" height="%s" alt="Gravatar" />'''