Merge pull request #262 from okfn/262-improve-helper-imports

Improve imports from lib.helpers

ckan/common.py

@@ -0,0 +1,17 @@
+# This file contains commonly used parts of external libraries. The idea is
+# to help in removing helpers from being used as a dependency by many files
+# but at the same time making it easy to change for example the json lib
+# used.
+#
+# NOTE:  This file is specificaly created for
+# from ckan.common import x, y, z to be allowed
+
+
+from pylons.i18n import _, ungettext
+from pylons import g, c, request, session, response
+import simplejson as json
+
+try:
+    from collections import OrderedDict  # from python 2.7
+except ImportError:
+    from sqlalchemy.util import OrderedDict

ckan/controllers/feed.py

@@ -29,7 +29,7 @@
 
 from ckan import model
 from ckan.lib.base import BaseController, c, request, response, json, abort, g
-from ckan.lib.helpers import date_str_to_datetime, url_for
+import ckan.lib.helpers as h
 from ckan.logic import get_action, NotFound
 
 # TODO make the item list configurable
@@ -348,18 +348,18 @@ def output_feed(self, results, feed_title, feed_description,
         for pkg in results:
             feed.add_item(
                 title=pkg.get('title', ''),
-                link=self.base_url + url_for(controller='package',
+                link=self.base_url + h.url_for(controller='package',
                                              action='read',
                                              id=pkg['id']),
                 description=pkg.get('notes', ''),
-                updated=date_str_to_datetime(pkg.get('metadata_modified')),
-                published=date_str_to_datetime(pkg.get('metadata_created')),
+                updated=h.date_str_to_datetime(pkg.get('metadata_modified')),
+                published=h.date_str_to_datetime(pkg.get('metadata_created')),
                 unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']),
                 author_name=pkg.get('author', ''),
                 author_email=pkg.get('author_email', ''),
                 categories=[t['name'] for t in pkg.get('tags', [])],
                 enclosure=webhelpers.feedgenerator.Enclosure(
-                    self.base_url + url_for(controller='api',
+                    self.base_url + h.url_for(controller='api',
                                             register='package',
                                             action='show',
                                             id=pkg['name'],
@@ -377,7 +377,7 @@ def _feed_url(self, query, controller, action, **kwargs):
         Constructs the url for the given action.  Encoding the query
         parameters.
         """
-        path = url_for(controller=controller, action=action, **kwargs)
+        path = h.url_for(controller=controller, action=action, **kwargs)
         query = [(k, v.encode('utf-8') if isinstance(v, basestring)
                   else str(v)) for k, v in query.items()]
 

ckan/controllers/group.py

@@ -4,11 +4,11 @@
 import datetime
 from urllib import urlencode
 
-from ckan.lib.base import BaseController, c, model, request, render, h, g
+from ckan.lib.base import BaseController, c, model, request, render, g
 from ckan.lib.base import ValidationException, abort, gettext
 import ckan.lib.base as base
 from pylons.i18n import get_lang, _
-from ckan.lib.helpers import Page
+import ckan.lib.helpers as h
 import ckan.lib.maintain as maintain
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
 from ckan.logic import NotFound, NotAuthorized, ValidationError
@@ -152,7 +152,7 @@ def index(self):
 
         results = self._action('group_list')(context, data_dict)
 
-        c.page = Page(
+        c.page = h.Page(
             collection=results,
             page=request.params.get('page', 1),
             url=h.pager_url,

ckan/controllers/package.py

@@ -9,17 +9,15 @@
 from paste.deploy.converters import asbool
 
 from ckan.logic import get_action, check_access
-from ckan.lib.helpers import date_str_to_datetime
 from ckan.lib.base import (request,
                            render,
                            BaseController,
                            model,
-                           abort, h, g, c)
+                           abort, g, c)
 from ckan.lib.base import response, redirect, gettext
 import ckan.lib.maintain as maintain
 from ckan.lib.package_saver import PackageSaver, ValidationException
 from ckan.lib.navl.dictization_functions import DataError, unflatten, validate
-from ckan.lib.helpers import json
 from ckan.logic import NotFound, NotAuthorized, ValidationError
 from ckan.logic import (tuplize_dict,
                         clean_dict,
@@ -339,7 +337,7 @@ def read(self, id, format='html'):
                 context['revision_id'] = revision_ref
             else:
                 try:
-                    date = date_str_to_datetime(revision_ref)
+                    date = h.date_str_to_datetime(revision_ref)
                     context['revision_date'] = date
                 except TypeError, e:
                     abort(400, _('Invalid revision format: %r') % e.args)
@@ -493,7 +491,7 @@ def new(self, data=None, errors=None, error_summary=None):
 
         data = data or clean_dict(unflatten(tuplize_dict(parse_params(
             request.params, ignore_keys=CACHE_PARAMETERS))))
-        c.resources_json = json.dumps(data.get('resources', []))
+        c.resources_json = h.json.dumps(data.get('resources', []))
         # convert tags if not supplied in data
         if data and not data.get('tag_string'):
             data['tag_string'] = ', '.join(
@@ -517,7 +515,7 @@ def new(self, data=None, errors=None, error_summary=None):
         vars = {'data': data, 'errors': errors,
                 'error_summary': error_summary,
                 'action': 'new', 'stage': stage}
-        c.errors_json = json.dumps(errors)
+        c.errors_json = h.json.dumps(errors)
 
         self._setup_template_variables(context, {},
                                        package_type=package_type)
@@ -770,7 +768,7 @@ def edit(self, id, data=None, errors=None, error_summary=None):
                             error_summary=error_summary)
 
         c.pkg = context.get("package")
-        c.resources_json = json.dumps(data.get('resources', []))
+        c.resources_json = h.json.dumps(data.get('resources', []))
 
         try:
             check_access('package_update', context)
@@ -783,7 +781,7 @@ def edit(self, id, data=None, errors=None, error_summary=None):
         errors = errors or {}
         vars = {'data': data, 'errors': errors,
                 'error_summary': error_summary, 'action': 'edit'}
-        c.errors_json = json.dumps(errors)
+        c.errors_json = h.json.dumps(errors)
 
         self._setup_template_variables(context, {'id': id},
                                        package_type=package_type)
@@ -830,7 +828,7 @@ def read_ajax(self, id, revision=None):
         data.pop('tags')
         data = flatten_to_string_key(data)
         response.headers['Content-Type'] = 'application/json;charset=utf-8'
-        return json.dumps(data)
+        return h.json.dumps(data)
 
     def history_ajax(self, id):
 
@@ -861,7 +859,7 @@ def history_ajax(self, id):
                          'current_approved': current_approved})
 
         response.headers['Content-Type'] = 'application/json;charset=utf-8'
-        return json.dumps(data)
+        return h.json.dumps(data)
 
     def _get_package_type(self, id):
         """
@@ -1317,7 +1315,7 @@ def resource_embedded_dataviewer(self, id, resource_id,
             c.resource = get_action('resource_show')(context,
                                                      {'id': resource_id})
             c.package = get_action('package_show')(context, {'id': id})
-            c.resource_json = json.dumps(c.resource)
+            c.resource_json = h.json.dumps(c.resource)
 
             # double check that the resource belongs to the specified package
             if not c.resource['id'] in [r['id']
@@ -1336,7 +1334,7 @@ def resource_embedded_dataviewer(self, id, resource_id,
             abort(400, ('"state" parameter must be a valid recline '
                         'state (version %d)' % state_version))
 
-        c.recline_state = json.dumps(recline_state)
+        c.recline_state = h.json.dumps(recline_state)
 
         c.width = max(int(request.params.get('width', width)), 100)
         c.height = max(int(request.params.get('height', height)), 100)
@@ -1352,7 +1350,7 @@ def _parse_recline_state(self, params):
         recline_state = {}
         for k, v in request.params.items():
             try:
-                v = json.loads(v)
+                v = h.json.loads(v)
             except ValueError:
                 pass
             recline_state[k] = v
@@ -1416,7 +1414,7 @@ def resource_datapreview(self, id, resource_id):
             plugin = plugins_that_can_preview[0]
             plugin.setup_template_variables(context, data_dict)
 
-            c.resource_json = json.dumps(c.resource)
+            c.resource_json = h.json.dumps(c.resource)
 
         except NotFound:
             abort(404, _('Resource not found'))

ckan/lib/base.py

@@ -24,8 +24,8 @@
 import ckan.lib.helpers as h
 import ckan.lib.app_globals as app_globals
 from ckan.plugins import PluginImplementations, IGenshiStreamFilter
-from ckan.lib.helpers import json
 import ckan.model as model
+from ckan.common import json
 
 log = logging.getLogger(__name__)
 

ckan/lib/dumper.py

@@ -4,7 +4,7 @@
 
 import ckan.model as model
 import ckan.model
-from helpers import json, OrderedDict
+from ckan.common import json, OrderedDict
 
 class SimpleDumper(object):
     '''Dumps just package data but including tags, groups, license text etc'''

ckan/lib/field_types.py

@@ -6,7 +6,8 @@
 with warnings.catch_warnings():
     warnings.filterwarnings('ignore', '.*compile_mappers.*')
     import formalchemy
-from ckan.lib.helpers import OrderedDict
+
+from ckan.common import OrderedDict
 
 months = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
 

ckan/lib/helpers.py

@@ -30,34 +30,22 @@
 from alphabet_paginate import AlphaPage
 import i18n
 import ckan.exceptions
-from pylons import request
-from pylons import session
-from pylons import c, g
-from pylons.i18n import _, ungettext
 
 import ckan.lib.fanstatic_resources as fanstatic_resources
 import ckan.model as model
 import ckan.lib.formatters as formatters
 import ckan.lib.maintain as maintain
 import ckan.lib.datapreview as datapreview
 
+from ckan.common import (
+    _, ungettext, g, c, request, session, json, OrderedDict
+)
+
 get_available_locales = i18n.get_available_locales
 get_locales_dict = i18n.get_locales_dict
 
 log = logging.getLogger(__name__)
 
-try:
-    from collections import OrderedDict  # from python 2.7
-except ImportError:
-    from sqlalchemy.util import OrderedDict
-
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-_log = logging.getLogger(__name__)
-
 
 def redirect_to(*args, **kw):
     '''A routes.redirect_to wrapper to retain the i18n settings'''

ckan/lib/jsonp.py

@@ -1,8 +1,6 @@
 import decorator
 
-from pylons import request, response
-
-from ckan.lib.helpers import json
+from ckan.common import json, request, response
 
 
 def to_jsonp(data):

ckan/lib/mailer.py

@@ -10,7 +10,7 @@
 from pylons.i18n.translation import _
 from pylons import config, g
 from ckan import model, __version__
-from ckan.lib.helpers import url_for
+import ckan.lib.helpers as h
 import paste.deploy.converters
 
 log = logging.getLogger(__name__)
@@ -115,7 +115,7 @@ def create_reset_key(user):
 
 def get_reset_link(user):
     return urljoin(g.site_url,
-                   url_for(controller='user',
+                   h.url_for(controller='user',
                            action='perform_reset',
                            id=user.id,
                            key=user.reset_key))

ckan/lib/repoze_patch.py

@@ -2,7 +2,7 @@
 from openid.consumer import consumer
 from openid.extensions import sreg, ax
 
-import lib.helpers as h
+import ckan.lib.helpers as h
 
 # #1659 fix - logged_out_url prefixed with mount point
 def get_full_path(path, environ):

ckan/lib/search/query.py

@@ -6,10 +6,11 @@
 from paste.deploy.converters import asbool
 from paste.util.multidict import MultiDict
 
+from ckan.common import json
+from ckan.lib.search.common import make_connection, SearchError, SearchQueryError
 import ckan.logic as logic
-from ckan import model
-from ckan.lib.helpers import json
-from common import make_connection, SearchError, SearchQueryError
+import ckan.model as model
+
 log = logging.getLogger(__name__)
 
 _open_licenses = None

ckan/logic/validators.py

@@ -6,7 +6,7 @@
 
 from ckan.lib.navl.dictization_functions import Invalid, StopOnError, Missing, missing, unflatten
 from ckan.logic import check_access, NotAuthorized, NotFound
-from ckan.lib.helpers import date_str_to_datetime
+import ckan.lib.helpers as h
 from ckan.model import (MAX_TAG_LENGTH, MIN_TAG_LENGTH,
                         PACKAGE_NAME_MIN_LENGTH, PACKAGE_NAME_MAX_LENGTH,
                         PACKAGE_VERSION_MAX_LENGTH,
@@ -67,7 +67,7 @@ def isodate(value, context):
     if value == '':
         return None
     try:
-        date = date_str_to_datetime(value)
+        date = h.date_str_to_datetime(value)
     except (TypeError, ValueError), e:
         raise Invalid(_('Date format incorrect'))
     return date

ckan/migration/versions/022_add_group_extras.py

@@ -5,7 +5,8 @@
 import uuid
 from sqlalchemy import types
 
-from ckan.lib.helpers import json
+from ckan.common import json
+
 class JsonType(types.TypeDecorator):
     '''Store data as JSON serializing on save and unserializing on use.
     '''

ckan/migration/versions/029_version_groups.py

@@ -1,14 +1,14 @@
+import uuid
+
 from sqlalchemy import *
 from sqlalchemy import types
 from migrate import *
 from datetime import datetime
 import migrate.changeset
-import uuid
 from migrate.changeset.constraint import ForeignKeyConstraint
 
+from ckan.common import json
 
-
-from ckan.lib.helpers import json
 class JsonType(types.TypeDecorator):
     '''Store data as JSON serializing on save and unserializing on use.
     '''

ckan/tests/__init__.py

@@ -27,12 +27,15 @@
 
 from ckan.lib.create_test_data import CreateTestData
 from ckan.lib import search
-from ckan.lib.helpers import _flash, url_for
-from ckan.lib.helpers import json
+import ckan.lib.helpers as h
 from ckan.logic import get_action
 from ckan.logic.action import get_domain_object
 import ckan.model as model
 from ckan import ckan_nose_plugin
+from ckan.common import json
+
+# evil hack as url_for is passed out
+url_for = h.url_for
 
 __all__ = ['url_for',
            'TestController',
@@ -349,7 +352,7 @@ def skip_test(*args):
     return test
 
 def clear_flash(res=None):
-    messages = _flash.pop_messages()
+    messages = h._flash.pop_messages()
 
 try:
     from nose.tools import assert_in, assert_not_in