[1407] WIP - Remove resourcegroups.

Contains a migration to remove the resourcegroups and have the resource
have a reference direct to it's package, this removes a lot of joins.

Drop constraint on resource_group_revision that travis complains about (but local install does not)

ckan/controllers/revision.py

@@ -62,8 +62,6 @@ def list(self):
                 package_indications = []
                 revision_changes = model.repo.list_changes(revision)
                 resource_revisions = revision_changes[model.Resource]
-                resource_group_revisions = \
-                    revision_changes[model.ResourceGroup]
                 package_extra_revisions = revision_changes[model.PackageExtra]
                 for package in revision.packages:
                     if not package:
@@ -89,16 +87,9 @@ def list(self):
                     else:
                         transition = 'updated'
                         for resource_revision in resource_revisions:
-                            if resource_revision.continuity.resource_group.\
-                                    package_id == package.id:
+                            if resource_revision.package_id == package.id:
                                 transition += ':resources'
                                 break
-                        for resource_group_revision in \
-                                resource_group_revisions:
-                            if resource_group_revision.package_id == \
-                                    package.id:
-                                transition += ':resource_group'
-                                break
                         for package_extra_revision in package_extra_revisions:
                             if package_extra_revision.package_id == \
                                     package.id:

ckan/lib/create_test_data.py

@@ -440,8 +440,8 @@ def create(cls, auth_profile="", package_type=None):
             )
         model.Session.add(pr1)
         model.Session.add(pr2)
-        pkg1.resource_groups_all[0].resources_all.append(pr1)
-        pkg1.resource_groups_all[0].resources_all.append(pr2)
+        pkg1.resources_all.append(pr1)
+        pkg1.resources_all.append(pr2)
         pkg1.notes = u'''Some test notes
 
 ### A 3rd level heading

ckan/lib/dictization/model_dictize.py

@@ -146,7 +146,6 @@ def _unified_resource_format(format_):
 def resource_dictize(res, context):
     model = context['model']
     resource = d.table_dictize(res, context)
-    resource_group_id = resource['resource_group_id']
     extras = resource.pop("extras", None)
     if extras:
         resource.update(extras)
@@ -156,13 +155,11 @@ def resource_dictize(res, context):
     ## for_edit is only called at the times when the dataset is to be edited
     ## in the frontend. Without for_edit the whole qualified url is returned.
     if resource.get('url_type') == 'upload' and not context.get('for_edit'):
-        resource_group = model.Session.query(
-            model.ResourceGroup).get(resource_group_id)
         last_part = url.split('/')[-1]
         cleaned_name = munge.munge_filename(last_part)
         resource['url'] = h.url_for(controller='package',
                                     action='resource_download',
-                                    id=resource_group.package_id,
+                                    id=resource['package_id'],
                                     resource_id=res.id,
                                     filename=cleaned_name,
                                     qualified=True)
@@ -238,14 +235,12 @@ def package_dictize(pkg, context):
     #strip whitespace from title
     if result_dict.get('title'):
         result_dict['title'] = result_dict['title'].strip()
+
     #resources
-    res_rev = model.resource_revision_table
-    resource_group = model.resource_group_table
-    q = select([res_rev], from_obj = res_rev.join(resource_group,
-               resource_group.c.id == res_rev.c.resource_group_id))
-    q = q.where(resource_group.c.package_id == pkg.id)
-    result = _execute_with_revision(q, res_rev, context)
-    result_dict["resources"] = resource_list_dictize(result, context)
+    resource_rev = model.resource_revision_table
+    q = select([resource_rev]).where(resource_rev.c.package_id == pkg.id)
+    result = _execute_with_revision(q, resource_rev, context)
+    result_dict["resources"] = resource_list_dictize(result or [], context)
     result_dict['num_resources'] = len(result_dict.get('resources', []))
 
     #tags

ckan/lib/dictization/model_save.py

@@ -71,8 +71,8 @@ def package_resource_list_save(res_dicts, package, context):
 
     pending = context.get('pending')
 
-    resource_list = package.resource_groups_all[0].resources_all
-    old_list = package.resource_groups_all[0].resources_all[:]
+    resource_list = package.resources_all
+    old_list = package.resources_all[:]
 
     obj_list = []
     for res_dict in res_dicts or []:

ckan/logic/action/get.py

@@ -1676,7 +1676,7 @@ def resource_search(context, data_dict):
     offset = data_dict.get('offset')
     limit = data_dict.get('limit')
 
-    q = model.Session.query(model.Resource).join(model.ResourceGroup).join(model.Package)
+    q = model.Session.query(model.Resource).join(model.Package)
     q = q.filter(model.Package.state == 'active')
     q = q.filter(model.Package.private == False)
     q = q.filter(model.Resource.state == 'active')

ckan/logic/action/update.py

@@ -90,7 +90,7 @@ def make_latest_pending_package_active(context, data_dict):
     _make_latest_rev_active(context, q)
 
     #resources
-    for resource in pkg.resource_groups_all[0].resources_all:
+    for resource in pkg.resources_all:
         q = session.query(model.ResourceRevision).filter_by(id=resource.id)
         _make_latest_rev_active(context, q)
 
@@ -214,7 +214,7 @@ def resource_update(context, data_dict):
     _check_access('resource_update', context, data_dict)
     del context["resource"]
 
-    package_id = resource.resource_group.package.id
+    package_id = resource.package.id
     pkg_dict = _get_action('package_show')(context, {'id': package_id})
 
     for n, p in enumerate(pkg_dict['resources']):
@@ -247,7 +247,7 @@ def package_update(context, data_dict):
 
     You must be authorized to edit the dataset and the groups that it belongs
     to.
-    
+
     It is recommended to call
     :py:func:`ckan.logic.action.get.package_show`, make the desired changes to
     the result, and then call ``package_update()`` with it.
@@ -735,7 +735,7 @@ def task_status_update_many(context, data_dict):
     '''Update many task statuses at once.
 
     :param data: the task_status dictionaries to update, for the format of task
-        status dictionaries see 
+        status dictionaries see
         :py:func:`~task_status_update`
     :type data: list of dictionaries
 

ckan/logic/auth/delete.py

@@ -26,11 +26,7 @@ def resource_delete(context, data_dict):
     resource = get_resource_object(context, data_dict)
 
     # check authentication against package
-    query = model.Session.query(model.Package)\
-        .join(model.ResourceGroup)\
-        .join(model.Resource)\
-        .filter(model.ResourceGroup.id == resource.resource_group_id)
-    pkg = query.first()
+    query = model.Package.get(resource.package_id)
     if not pkg:
         raise logic.NotFound(_('No package found for this resource, cannot check auth.'))
 

ckan/logic/auth/get.py

@@ -126,11 +126,7 @@ def resource_show(context, data_dict):
     resource = get_resource_object(context, data_dict)
 
     # check authentication against package
-    query = model.Session.query(model.Package)\
-        .join(model.ResourceGroup)\
-        .join(model.Resource)\
-        .filter(model.ResourceGroup.id == resource.resource_group_id)
-    pkg = query.first()
+    pkg = model.Package.get(resource.package_id)
     if not pkg:
         raise logic.NotFound(_('No package found for this resource, cannot check auth.'))
 

ckan/logic/auth/update.py

@@ -51,11 +51,7 @@ def resource_update(context, data_dict):
     resource = logic_auth.get_resource_object(context, data_dict)
 
     # check authentication against package
-    query = model.Session.query(model.Package)\
-        .join(model.ResourceGroup)\
-        .join(model.Resource)\
-        .filter(model.ResourceGroup.id == resource.resource_group_id)
-    pkg = query.first()
+    pkg = model.Package.get(resource.package_id)
     if not pkg:
         raise logic.NotFound(
             _('No package found for this resource, cannot check auth.')

ckan/logic/schema.py

@@ -67,7 +67,6 @@ def default_resource_schema():
     schema = {
         'id': [ignore_empty, unicode],
         'revision_id': [ignore_missing, unicode],
-        'resource_group_id': [ignore],
         'package_id': [ignore],
         'url': [not_empty, unicode],#, URL(add_http=False)],
         'description': [ignore_missing, unicode],
@@ -204,7 +203,6 @@ def default_show_package_schema():
         'cache_last_updated': [ckan.lib.navl.validators.ignore_missing],
         'webstore_last_updated': [ckan.lib.navl.validators.ignore_missing],
         'revision_timestamp': [],
-        'resource_group_id': [],
         'cache_last_updated': [],
         'webstore_last_updated': [],
         'size': [],

ckan/migration/versions/072-remove-resource-groups.py

@@ -0,0 +1,28 @@
+import ckan.model
+
+
+def upgrade(migrate_engine):
+    migrate_engine.execute(
+        '''
+        ALTER TABLE "resource" ADD COLUMN "package_id" text NOT NULL DEFAULT '';
+        UPDATE "resource" AS R
+        SET package_id = G.package_id
+        FROM "resource_group" AS G
+        WHERE R.resource_group_id = G.id;
+        ALTER TABLE "resource" DROP COLUMN "resource_group_id";
+
+        ALTER TABLE "resource_revision" ADD COLUMN "package_id" text NOT NULL DEFAULT '';
+        UPDATE "resource_revision" AS R
+        SET package_id = G.package_id
+        FROM "resource_group_revision" AS G
+        WHERE R.resource_group_id = G.id;
+        ALTER TABLE "resource_revision" DROP COLUMN "resource_group_id";
+
+        ALTER TABLE resource_group_revision DROP CONSTRAINT resource_group_revision_continuity_id_fkey;
+
+        DROP TABLE "resource_group_revision";
+        DROP TABLE "resource_group";
+        '''
+    )
+
+

ckan/model/__init__.py

@@ -89,14 +89,10 @@
 )
 from resource import (
     Resource,
-    ResourceGroup,
     ResourceRevision,
     DictProxy,
-    resource_group_table,
     resource_table,
     resource_revision_table,
-    ResourceGroupRevision,
-    resource_group_revision_table,
 )
 from tracking import (
     tracking_summary_table,
@@ -408,7 +404,7 @@ def purge_revision(self, revision, leave_record=False):
 
 repo = Repository(meta.metadata, meta.Session,
                   versioned_objects=[Package, PackageTag, Resource,
-                                     ResourceGroup, PackageExtra, Member,
+                                     PackageExtra, Member,
                                      Group]
         )
 

ckan/model/package.py

@@ -65,8 +65,6 @@ class Package(vdm.sqlalchemy.RevisionedObjectMixin,
     def __init__(self, **kw):
         from ckan import model
         super(Package, self).__init__(**kw)
-        resource_group = model.ResourceGroup(label="default")
-        self.resource_groups_all.append(resource_group)
 
     @classmethod
     def search_by_name(cls, text_query):
@@ -85,21 +83,17 @@ def get(cls, reference):
 
     @property
     def resources(self):
-        if len(self.resource_groups_all) == 0:
-            return []
-
-        assert len(self.resource_groups_all) == 1, "can only use resources on packages if there is only one resource_group"
-        return [resource for resource in 
-                self.resource_groups_all[0].resources_all
+        return [resource for resource in
+                self.resources_all
                 if resource.state <> 'deleted']
 
     def related_packages(self):
         return [self]
 
     def add_resource(self, url, format=u'', description=u'', hash=u'', **kw):
         import resource
-        self.resource_groups_all[0].resources_all.append(resource.Resource(
-            resource_group_id=self.resource_groups_all[0].id,
+        self.resources_all.append(resource.Resource(
+            package_id=self.id,
             url=url,
             format=format,
             description=description,
@@ -377,22 +371,17 @@ def all_related_revisions(self):
                 Ordered by most recent first.
         '''
         from tag import PackageTag
-        from resource import ResourceGroup, Resource
+        from resource import Resource
         from package_extra import PackageExtra
 
         results = {} # revision:[PackageRevision1, PackageTagRevision1, etc.]
         for pkg_rev in self.all_revisions:
             if not results.has_key(pkg_rev.revision):
                 results[pkg_rev.revision] = []
             results[pkg_rev.revision].append(pkg_rev)
-        for class_ in [ResourceGroup, Resource, PackageExtra, PackageTag]:
+        for class_ in [Resource, PackageExtra, PackageTag]:
             rev_class = class_.__revision_class__
-            if class_ == Resource:
-                q = meta.Session.query(rev_class).join('continuity',
-                                                  'resource_group')
-                obj_revisions = q.filter(ResourceGroup.package_id == self.id).all()
-            else:
-                obj_revisions = meta.Session.query(rev_class).filter_by(package_id=self.id).all()
+            obj_revisions = meta.Session.query(rev_class).filter_by(package_id=self.id).all()
             for obj_rev in obj_revisions:
                 if not results.has_key(obj_rev.revision):
                     results[obj_rev.revision] = []
@@ -413,32 +402,23 @@ def diff(self, to_revision=None, from_revision=None):
         '''Overrides the diff in vdm, so that related obj revisions are
         diffed as well as PackageRevisions'''
         from tag import PackageTag
-        from resource import ResourceGroup, Resource
+        from resource import Resource
         from package_extra import PackageExtra
 
         results = {} # field_name:diffs
         results.update(super(Package, self).diff(to_revision, from_revision))
         # Iterate over PackageTag, PackageExtra, Resources etc.
-        for obj_class in [ResourceGroup, Resource, PackageExtra, PackageTag]:
+        for obj_class in [Resource, PackageExtra, PackageTag]:
             obj_rev_class = obj_class.__revision_class__
             # Query for object revisions related to this package
-            if obj_class == Resource:
-                obj_rev_query = meta.Session.query(obj_rev_class).\
-                                join('continuity', 'resource_group').\
-                                join('revision').\
-                                filter(ResourceGroup.package_id == self.id).\
-                                order_by(core.Revision.timestamp.desc())
-            else:
-                obj_rev_query = meta.Session.query(obj_rev_class).\
-                                filter_by(package_id=self.id).\
-                                join('revision').\
-                                order_by(core.Revision.timestamp.desc())
+            obj_rev_query = meta.Session.query(obj_rev_class).\
+                            filter_by(package_id=self.id).\
+                            join('revision').\
+                            order_by(core.Revision.timestamp.desc())
             # Columns to include in the diff
             cols_to_diff = obj_class.revisioned_fields()
             cols_to_diff.remove('id')
             if obj_class is Resource:
-                cols_to_diff.remove('resource_group_id')
-            else:
                 cols_to_diff.remove('package_id')
             # Particular object types are better known by an invariant field
             if obj_class is PackageTag: