Merge remote-tracking branch 'origin/master' into 2003-package-patch-…

…simple-version

ckan/controllers/revision.py

@@ -62,8 +62,6 @@ def list(self):
                 package_indications = []
                 revision_changes = model.repo.list_changes(revision)
                 resource_revisions = revision_changes[model.Resource]
-                resource_group_revisions = \
-                    revision_changes[model.ResourceGroup]
                 package_extra_revisions = revision_changes[model.PackageExtra]
                 for package in revision.packages:
                     if not package:
@@ -89,16 +87,9 @@ def list(self):
                     else:
                         transition = 'updated'
                         for resource_revision in resource_revisions:
-                            if resource_revision.continuity.resource_group.\
-                                    package_id == package.id:
+                            if resource_revision.package_id == package.id:
                                 transition += ':resources'
                                 break
-                        for resource_group_revision in \
-                                resource_group_revisions:
-                            if resource_group_revision.package_id == \
-                                    package.id:
-                                transition += ':resource_group'
-                                break
                         for package_extra_revision in package_extra_revisions:
                             if package_extra_revision.package_id == \
                                     package.id:

ckan/lib/base.py

@@ -130,12 +130,6 @@ def render_template():
         except render_.TemplateNotFound:
             raise
 
-        # snippets should not pass the context
-        # but allow for legacy genshi templates
-        if renderer == 'snippet' and template_type != 'genshi':
-            del globs['c']
-            del globs['tmpl_context']
-
         log.debug('rendering %s [%s]' % (template_path, template_type))
         if config.get('debug'):
             context_vars = globs.get('c')

ckan/lib/create_test_data.py

@@ -440,8 +440,8 @@ def create(cls, auth_profile="", package_type=None):
             )
         model.Session.add(pr1)
         model.Session.add(pr2)
-        pkg1.resource_groups_all[0].resources_all.append(pr1)
-        pkg1.resource_groups_all[0].resources_all.append(pr2)
+        pkg1.resources_all.append(pr1)
+        pkg1.resources_all.append(pr2)
         pkg1.notes = u'''Some test notes
 
 ### A 3rd level heading

ckan/lib/dictization/model_dictize.py

@@ -108,7 +108,6 @@ def extras_list_dictize(extras_list, context):
 def resource_dictize(res, context):
     model = context['model']
     resource = d.table_dictize(res, context)
-    resource_group_id = resource['resource_group_id']
     extras = resource.pop("extras", None)
     if extras:
         resource.update(extras)
@@ -117,13 +116,11 @@ def resource_dictize(res, context):
     ## for_edit is only called at the times when the dataset is to be edited
     ## in the frontend. Without for_edit the whole qualified url is returned.
     if resource.get('url_type') == 'upload' and not context.get('for_edit'):
-        resource_group = model.Session.query(
-            model.ResourceGroup).get(resource_group_id)
         last_part = url.split('/')[-1]
         cleaned_name = munge.munge_filename(last_part)
         resource['url'] = h.url_for(controller='package',
                                     action='resource_download',
-                                    id=resource_group.package_id,
+                                    id=resource['package_id'],
                                     resource_id=res.id,
                                     filename=cleaned_name,
                                     qualified=True)
@@ -199,13 +196,11 @@ def package_dictize(pkg, context):
     #strip whitespace from title
     if result_dict.get('title'):
         result_dict['title'] = result_dict['title'].strip()
+
     #resources
-    res_rev = model.resource_revision_table
-    resource_group = model.resource_group_table
-    q = select([res_rev], from_obj = res_rev.join(resource_group,
-               resource_group.c.id == res_rev.c.resource_group_id))
-    q = q.where(resource_group.c.package_id == pkg.id)
-    result = _execute_with_revision(q, res_rev, context)
+    resource_rev = model.resource_revision_table
+    q = select([resource_rev]).where(resource_rev.c.package_id == pkg.id)
+    result = _execute_with_revision(q, resource_rev, context)
     result_dict["resources"] = resource_list_dictize(result, context)
     result_dict['num_resources'] = len(result_dict.get('resources', []))
 
@@ -737,7 +732,7 @@ def resource_view_dictize(resource_view, context):
     config = dictized.pop('config', {})
     dictized.update(config)
     resource = context['model'].Resource.get(resource_view.resource_id)
-    package_id = resource.resource_group.package_id
+    package_id = resource.package_id
     dictized['package_id'] = package_id
     return dictized
 

ckan/lib/dictization/model_save.py

@@ -65,11 +65,13 @@ def package_resource_list_save(res_dicts, package, context):
 
     pending = context.get('pending')
 
-    resource_list = package.resource_groups_all[0].resources_all
-    old_list = package.resource_groups_all[0].resources_all[:]
+    resource_list = package.resources_all
+    old_list = package.resources_all[:]
 
     obj_list = []
     for res_dict in res_dicts or []:
+        if not u'package_id' in res_dict or not res_dict[u'package_id']:
+            res_dict[u'package_id'] = package.id
         obj = resource_dict_save(res_dict, context)
         obj_list.append(obj)
 

ckan/lib/plugins.py

@@ -263,9 +263,15 @@ def search_template(self):
     def history_template(self):
         return 'package/history.html'
 
+    def resource_template(self):
+        return 'package/resource_read.html'
+
     def package_form(self):
         return 'package/new_package_form.html'
 
+    def resource_form(self):
+        return 'package/snippets/resource_form.html'
+
 
 class DefaultGroupForm(object):
     """

ckan/lib/render.py

@@ -1,8 +1,11 @@
 import os
 import re
+import logging
 
 from pylons import config
 
+log = logging.getLogger(__name__)
+
 _template_info_cache = {}
 
 def reset_template_info_cache():
@@ -58,3 +61,45 @@ def template_info(template_name):
                   'template_type' : t_type,}
         _template_info_cache[template_name] = t_data
     return template_path, t_type
+
+
+def deprecated_lazy_render(target_template, snippet_template, render,
+        message):
+    '''
+    If either target_template is a genshi template this function renders
+    immediately and returns a string, otherwise an object that will lazily
+    render when {{ the_object | safe }} is used in the jinja template
+    is returned.
+
+    :param target_template: name of jinja or genshi template
+        that may include the object returned
+    :param snippet_template: name of jinja or genshi template
+        that will be rendered
+    :param render: function to call to render the template snippet
+    :param message: message to log.warn() if render is called
+    '''
+    lazy = DeprecatedLazyRenderer(render, message)
+    t_path, t_type = template_info(target_template)
+    if t_type.startswith('genshi'):
+        return lazy.__html__()
+    return lazy
+
+class DeprecatedLazyRenderer(object):
+    '''
+    An object that will defer rendering until absolutely necessary to
+    pass to a template that might do {{ the_object | safe }}.
+
+    Used maintain backwards compatibility with templates that used to
+    expect a pre-rendered HTML snippet but have been updated to use
+    a normal {% snippet %} tag.
+    '''
+    def __init__(self, render, message):
+        self._html = None
+        self._render = render
+        self._message = message
+
+    def __html__(self):
+        if not self._html:
+            log.warn(self._message)
+            self._html = self._render()
+        return self._html

ckan/lib/search/index.py

@@ -24,8 +24,6 @@
 
 log = logging.getLogger(__name__)
 
-_validate = ckan.lib.navl.dictization_functions.validate
-
 TYPE_FIELD = "entity_type"
 PACKAGE_TYPE = "package"
 KEY_CHARS = string.digits + string.letters + "_-"
@@ -113,8 +111,9 @@ def index_package(self, pkg_dict, defer_commit=False):
                 pkg_dict.get('type'))
 
             schema = package_plugin.show_package_schema()
-            validated_pkg_dict, errors = _validate(pkg_dict, schema, {
-                'model': model, 'session': model.Session})
+            validated_pkg_dict, errors = lib_plugins.plugin_validate(
+                package_plugin, {'model': model, 'session': model.Session},
+                pkg_dict, schema, 'package_show')
             pkg_dict['validated_data_dict'] = json.dumps(validated_pkg_dict,
                 cls=ckan.lib.navl.dictization_functions.MissingNullEncoder)
 

ckan/logic/action/create.py

@@ -217,6 +217,7 @@ def resource_create(context, data_dict):
     '''Appends a new resource to a datasets list of resources.
 
     :param package_id: id of package that the resource should be added to.
+
     :type package_id: string
     :param url: url of resource
     :type url: string
@@ -582,8 +583,8 @@ def _group_or_org_create(context, data_dict, is_org=False):
     upload.update_data_dict(data_dict, 'image_url',
                             'image_upload', 'clear_upload')
     # get the schema
-    group_plugin = lib_plugins.lookup_group_plugin(
-        group_type=data_dict.get('type'))
+    group_type = data_dict.get('type')
+    group_plugin = lib_plugins.lookup_group_plugin(group_type)
     try:
         schema = group_plugin.form_to_db_schema_options({
             'type': 'create', 'api': 'api_version' in context,

ckan/logic/action/get.py

@@ -1021,7 +1021,7 @@ def resource_show(context, data_dict):
 
     pkg_dict = logic.get_action('package_show')(
         dict(context),
-        {'id': resource.resource_group.package.id})
+        {'id': resource.package.id})
 
     for resource_dict in pkg_dict['resources']:
         if resource_dict['id'] == id:
@@ -1030,6 +1030,8 @@ def resource_show(context, data_dict):
         logging.error('Could not find resource ' + id)
         raise NotFound(_('Resource was not found.'))
 
+    resource_dict['package_id'] = pkg_dict['id']
+
     # original dictized version didn't include this field:
     resource_dict.pop('revision_timestamp')
     return resource_dict
@@ -1874,12 +1876,11 @@ def resource_search(context, data_dict):
     offset = data_dict.get('offset')
     limit = data_dict.get('limit')
 
-    q = (model.Session.query(model.Resource)
-         .join(model.ResourceGroup)
-         .join(model.Package))
-    q = q.filter(model.Package.state == 'active')
-    q = q.filter(model.Package.private == False)
-    q = q.filter(model.Resource.state == 'active')
+    q = model.Session.query(model.Resource) \
+         .join(model.Package) \
+         .filter(model.Package.state == 'active') \
+         .filter(model.Package.private == False) \
+         .filter(model.Resource.state == 'active') \
 
     resource_fields = model.Resource.get_columns()
     for field, terms in fields.items():

ckan/logic/action/update.py

@@ -91,7 +91,7 @@ def make_latest_pending_package_active(context, data_dict):
     _make_latest_rev_active(context, q)
 
     #resources
-    for resource in pkg.resource_groups_all[0].resources_all:
+    for resource in pkg.resources_all:
         q = session.query(model.ResourceRevision).filter_by(id=resource.id)
         _make_latest_rev_active(context, q)
 
@@ -215,7 +215,7 @@ def resource_update(context, data_dict):
     _check_access('resource_update', context, data_dict)
     del context["resource"]
 
-    package_id = resource.resource_group.package.id
+    package_id = resource.package.id
     pkg_dict = _get_action('package_show')(context, {'id': package_id})
 
     for n, p in enumerate(pkg_dict['resources']):
@@ -377,6 +377,7 @@ def package_update(context, data_dict):
         raise NotFound(_('Package was not found.'))
     context["package"] = pkg
     data_dict["id"] = pkg.id
+    data_dict['type'] = pkg.type
 
     _check_access('package_update', context, data_dict)
 
@@ -599,6 +600,8 @@ def _group_or_org_update(context, data_dict, is_org=False):
     if group is None:
         raise NotFound('Group was not found.')
 
+    data_dict['type'] = group.type
+
     # get the schema
     group_plugin = lib_plugins.lookup_group_plugin(group.type)
     try:

ckan/logic/auth/delete.py

@@ -13,7 +13,7 @@ def user_delete(context, data_dict):
 
 
 def package_delete(context, data_dict):
-    # Defer auhtorization for package_delete to package_update, as deletions
+    # Defer authorization for package_delete to package_update, as deletions
     # are essentially changing the state field
     return _auth_update.package_update(context, data_dict)
 
@@ -23,11 +23,7 @@ def resource_delete(context, data_dict):
     resource = get_resource_object(context, data_dict)
 
     # check authentication against package
-    query = model.Session.query(model.Package)\
-        .join(model.ResourceGroup)\
-        .join(model.Resource)\
-        .filter(model.ResourceGroup.id == resource.resource_group_id)
-    pkg = query.first()
+    pkg = model.Package.get(resource.package_id)
     if not pkg:
         raise logic.NotFound(_('No package found for this resource, cannot check auth.'))
 

ckan/logic/auth/get.py

@@ -126,11 +126,7 @@ def resource_show(context, data_dict):
     resource = get_resource_object(context, data_dict)
 
     # check authentication against package
-    query = model.Session.query(model.Package)\
-        .join(model.ResourceGroup)\
-        .join(model.Resource)\
-        .filter(model.ResourceGroup.id == resource.resource_group_id)
-    pkg = query.first()
+    pkg = model.Package.get(resource.package_id)
     if not pkg:
         raise logic.NotFound(_('No package found for this resource, cannot check auth.'))
 

ckan/logic/auth/update.py

@@ -62,11 +62,7 @@ def resource_update(context, data_dict):
     resource = logic_auth.get_resource_object(context, data_dict)
 
     # check authentication against package
-    query = model.Session.query(model.Package)\
-        .join(model.ResourceGroup)\
-        .join(model.Resource)\
-        .filter(model.ResourceGroup.id == resource.resource_group_id)
-    pkg = query.first()
+    pkg = model.Package.get(resource.package_id)
     if not pkg:
         raise logic.NotFound(
             _('No package found for this resource, cannot check auth.')

ckan/logic/schema.py

@@ -71,8 +71,7 @@ def default_resource_schema():
 
     schema = {
         'id': [ignore_empty, unicode],
-        'revision_id': [ignore],
-        'resource_group_id': [ignore],
+        'revision_id': [ignore_missing, unicode],
         'package_id': [ignore],
         'url': [not_empty, unicode, remove_whitespace],
         'description': [ignore_missing, unicode],
@@ -213,7 +212,6 @@ def default_show_package_schema():
         'webstore_last_updated': [ckan.lib.navl.validators.ignore_missing],
         'revision_timestamp': [],
         'revision_id': [],
-        'resource_group_id': [],
         'cache_last_updated': [],
         'webstore_last_updated': [],
         'size': [],

ckan/migration/versions/074_remove_resource_groups.py

@@ -0,0 +1,29 @@
+import ckan.model
+
+
+def upgrade(migrate_engine):
+    migrate_engine.execute(
+        '''
+        ALTER TABLE "resource"
+            ADD COLUMN "package_id" text NOT NULL DEFAULT '';
+        UPDATE "resource" AS R
+        SET package_id = G.package_id
+        FROM "resource_group" AS G
+        WHERE R.resource_group_id = G.id;
+        ALTER TABLE "resource" DROP COLUMN "resource_group_id";
+
+        ALTER TABLE "resource_revision"
+            ADD COLUMN "package_id" text NOT NULL DEFAULT '';
+        UPDATE "resource_revision" AS R
+        SET package_id = G.package_id
+        FROM "resource_group_revision" AS G
+        WHERE R.resource_group_id = G.id;
+        ALTER TABLE "resource_revision" DROP COLUMN "resource_group_id";
+
+        ALTER TABLE resource_group_revision
+            DROP CONSTRAINT resource_group_revision_continuity_id_fkey;
+
+        DROP TABLE "resource_group_revision";
+        DROP TABLE "resource_group";
+        '''
+    )