Merge with master

ckan/config/resource_formats.json

@@ -10,9 +10,9 @@
   ],
   ["PPTX", "Powerpoint OOXML Presentation", "application/vnd.openxmlformats-officedocument.presentationml.presentation", []],
   ["EXE", "Windows Executable Program", "application/x-msdownload", []],
-  ["DOC", "Word Document", "application/ms-word", []],
+  ["DOC", "Word Document", "application/msword", []],
   ["KML", "KML File", "application/vnd.google-earth.kml+xml", []],
-  ["XLS", "Excel Document", "application/vnd.ms-excel", []],
+  ["XLS", "Excel Document", "application/vnd.ms-excel", ["Excel", "application/msexcel", "application/x-msexcel", "application/x-ms-excel", "application/x-excel", "application/x-dos_ms_excel", "application/xls", "application/x-xls"]],
   ["WCS", "Web Coverage Service", "wcs", []],
   ["JS", "JavaScript", "application/x-javascript", []],
   ["MDB", "Access Database", "application/x-msaccess", []],

ckan/lib/cli.py

@@ -890,7 +890,6 @@ class DatasetCmd(CkanCommand):
 
     def command(self):
         self._load_config()
-        import ckan.model as model
 
         if not self.args:
             print self.usage
@@ -939,13 +938,14 @@ def delete(self, dataset_ref):
         print '%s %s -> %s' % (dataset.name, old_state, dataset.state)
 
     def purge(self, dataset_ref):
-        import ckan.model as model
+        import ckan.logic as logic
         dataset = self._get_dataset(dataset_ref)
         name = dataset.name
 
-        rev = model.repo.new_revision()
-        dataset.purge()
-        model.repo.commit_and_remove()
+        site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {})
+        context = {'user': site_user['name']}
+        logic.get_action('dataset_purge')(
+            context, {'id': dataset_ref})
         print '%s purged' % name
 
 

ckan/logic/action/delete.py

@@ -47,6 +47,9 @@ def user_delete(context, data_dict):
 def package_delete(context, data_dict):
     '''Delete a dataset (package).
 
+    This makes the dataset disappear from all web & API views, apart from the
+    trash.
+
     You must be authorized to delete the dataset.
 
     :param id: the id or name of the dataset to delete
@@ -76,6 +79,46 @@ def package_delete(context, data_dict):
     entity.delete()
     model.repo.commit()
 
+
+def dataset_purge(context, data_dict):
+    '''Purge a dataset.
+
+    .. warning:: Purging a dataset cannot be undone!
+
+    Purging a database completely removes the dataset from the CKAN database,
+    whereas deleting a dataset simply marks the dataset as deleted (it will no
+    longer show up in the front-end, but is still in the db).
+
+    You must be authorized to purge the dataset.
+
+    :param id: the name or id of the dataset to be purged
+    :type id: string
+
+    '''
+    model = context['model']
+    id = _get_or_bust(data_dict, 'id')
+
+    pkg = model.Package.get(id)
+    context['package'] = pkg
+    if pkg is None:
+        raise NotFound('Dataset was not found')
+
+    _check_access('dataset_purge', context, data_dict)
+
+    members = model.Session.query(model.Member) \
+                   .filter(model.Member.table_id == pkg.id) \
+                   .filter(model.Member.table_name == 'package')
+    if members.count() > 0:
+        for m in members.all():
+            m.purge()
+
+    pkg = model.Package.get(id)
+    # no new_revision() needed since there are no object_revisions created
+    # during purge
+    pkg.purge()
+    model.repo.commit_and_remove()
+
+
 def resource_delete(context, data_dict):
     '''Delete a resource from a dataset.
 

ckan/logic/auth/delete.py

@@ -17,6 +17,10 @@ def package_delete(context, data_dict):
     # are essentially changing the state field
     return _auth_update.package_update(context, data_dict)
 
+def dataset_purge(context, data_dict):
+    # Only sysadmins are authorized to purge datasets
+    return {'success': False}
+
 def resource_delete(context, data_dict):
     model = context['model']
     user = context.get('user')

ckan/migration/versions/079_resource_revision_index.py

@@ -0,0 +1,7 @@
+def upgrade(migrate_engine):
+    migrate_engine.execute(
+        '''
+        CREATE INDEX idx_resource_continuity_id
+            ON resource_revision (continuity_id);
+        '''
+    )

ckan/migration/versions/080_continuity_id_indexes.py

@@ -0,0 +1,13 @@
+def upgrade(migrate_engine):
+    migrate_engine.execute(
+        '''
+        CREATE INDEX idx_member_continuity_id
+            ON member_revision (continuity_id);
+        CREATE INDEX idx_package_tag_continuity_id
+            ON package_tag_revision (continuity_id);
+        CREATE INDEX idx_package_continuity_id
+            ON package_revision (continuity_id);
+        CREATE INDEX idx_package_extra_continuity_id
+            ON package_extra_revision (continuity_id);
+        '''
+    )

ckan/model/domain_object.py

@@ -76,15 +76,12 @@ def remove(self):
         self.Session.remove()
 
     def delete(self):
+        # stateful objects have this method overridden - see
+        # vmd.base.StatefulObjectMixin
         self.Session.delete(self)
 
     def purge(self):
         self.Session().autoflush = False
-        if hasattr(self, '__revisioned__'): # only for versioned objects ...
-            # this actually should auto occur due to cascade relationships but
-            # ...
-            for rev in self.all_revisions:
-                self.Session.delete(rev)
         self.Session.delete(self)
 
     def as_dict(self):

ckan/model/group.py

@@ -104,11 +104,11 @@ def related_packages(self):
     def __unicode__(self):
         # refer to objects by name, not ID, to help debugging
         if self.table_name == 'package':
-            table_info = 'package=%s' % meta.Session.query(_package.Package).\
-                get(self.table_id).name
+            pkg = meta.Session.query(_package.Package).get(self.table_id)
+            table_info = 'package=%s' % pkg.name if pkg else 'None'
         elif self.table_name == 'group':
-            table_info = 'group=%s' % meta.Session.query(Group).\
-                get(self.table_id).name
+            group = meta.Session.query(Group).get(self.table_id)
+            table_info = 'group=%s' % group.name if group else 'None'
         else:
             table_info = 'table_name=%s table_id=%s' % (self.table_name,
                                                         self.table_id)

ckan/tests/lib/test_helpers.py

@@ -180,7 +180,21 @@ def test_includes_existing_license(self):
         eq_(licenses[0][0], 'some-old-license')
 
 
-class TestResourceFormat(object):
+class TestUnifiedResourceFormat(object):
+    def test_unified_resource_format_by_extension(self):
+        eq_(h.unified_resource_format('xls'), 'XLS')
+
+    def test_unified_resource_format_by_description(self):
+        eq_(h.unified_resource_format('Excel document'), 'XLS')
+
+    def test_unified_resource_format_by_primary_mimetype(self):
+        eq_(h.unified_resource_format('application/vnd.ms-excel'), 'XLS')
+
+    def test_unified_resource_format_by_alternative_description(self):
+        eq_(h.unified_resource_format('application/msexcel'), 'XLS')
+
+    def test_unified_resource_format_by_alternative_description2(self):
+        eq_(h.unified_resource_format('Excel'), 'XLS')
 
     def test_autodetect_tsv(self):
 

ckan/tests/logic/action/test_delete.py

@@ -53,7 +53,8 @@ def test_resource_view_delete(self):
 
         helpers.call_action('resource_view_delete', context={}, **params)
 
-        assert_raises(logic.NotFound, helpers.call_action, 'resource_view_show',
+        assert_raises(logic.NotFound, helpers.call_action,
+                      'resource_view_show',
                       context={}, **params)
 
         # The model object is actually deleted
@@ -64,14 +65,16 @@ def test_delete_no_id_raises_validation_error(self):
 
         params = {}
 
-        assert_raises(logic.ValidationError, helpers.call_action, 'resource_view_delete',
+        assert_raises(logic.ValidationError, helpers.call_action,
+                      'resource_view_delete',
                       context={}, **params)
 
     def test_delete_wrong_id_raises_not_found_error(self):
 
         params = {'id': 'does_not_exist'}
 
-        assert_raises(logic.NotFound, helpers.call_action, 'resource_view_delete',
+        assert_raises(logic.NotFound, helpers.call_action,
+                      'resource_view_delete',
                       context={}, **params)
 
 
@@ -346,3 +349,110 @@ def test_missing_id_returns_error(self):
     def test_bad_id_returns_404(self):
         assert_raises(logic.NotFound,
                       helpers.call_action, 'organization_purge', id='123')
+
+
+class TestDatasetPurge(object):
+    def setup(self):
+        helpers.reset_db()
+
+    def test_a_non_sysadmin_cant_purge_dataset(self):
+        user = factories.User()
+        dataset = factories.Dataset(user=user)
+
+        assert_raises(logic.NotAuthorized,
+                      helpers.call_action,
+                      'dataset_purge',
+                      context={'user': user['name'], 'ignore_auth': False},
+                      id=dataset['name'])
+
+    def test_purged_dataset_does_not_show(self):
+        dataset = factories.Dataset()
+
+        helpers.call_action('dataset_purge',
+                            context={'ignore_auth': True},
+                            id=dataset['name'])
+
+        assert_raises(logic.NotFound, helpers.call_action, 'package_show',
+                      context={}, id=dataset['name'])
+
+    def test_purged_dataset_is_not_listed(self):
+        dataset = factories.Dataset()
+
+        helpers.call_action('dataset_purge', id=dataset['name'])
+
+        assert_equals(helpers.call_action('package_list', context={}), [])
+
+    def test_group_no_longer_shows_its_purged_dataset(self):
+        group = factories.Group()
+        dataset = factories.Dataset(groups=[{'name': group['name']}])
+
+        helpers.call_action('dataset_purge', id=dataset['name'])
+
+        dataset_shown = helpers.call_action('group_show', context={},
+                                            id=group['id'],
+                                            include_datasets=True)
+        assert_equals(dataset_shown['packages'], [])
+
+    def test_purged_dataset_is_not_in_search_results(self):
+        search.clear()
+        dataset = factories.Dataset()
+
+        def get_search_results():
+            results = helpers.call_action('package_search',
+                                          q=dataset['title'])['results']
+            return [d['name'] for d in results]
+        assert_equals(get_search_results(), [dataset['name']])
+
+        helpers.call_action('dataset_purge', id=dataset['name'])
+
+        assert_equals(get_search_results(), [])
+
+    def test_purged_dataset_leaves_no_trace_in_the_model(self):
+        factories.Group(name='group1')
+        org = factories.Organization()
+        dataset = factories.Dataset(
+            tags=[{'name': 'tag1'}],
+            groups=[{'name': 'group1'}],
+            owner_org=org['id'],
+            extras=[{'key': 'testkey', 'value': 'testvalue'}])
+        factories.Resource(package_id=dataset['id'])
+        num_revisions_before = model.Session.query(model.Revision).count()
+
+        helpers.call_action('dataset_purge',
+                            context={'ignore_auth': True},
+                            id=dataset['name'])
+        num_revisions_after = model.Session.query(model.Revision).count()
+
+        # the Package and related objects are gone
+        assert_equals(model.Session.query(model.Package).all(), [])
+        assert_equals(model.Session.query(model.Resource).all(), [])
+        assert_equals(model.Session.query(model.PackageTag).all(), [])
+        # there is no clean-up of the tag object itself, just the PackageTag.
+        assert_equals([t.name for t in model.Session.query(model.Tag).all()],
+                      ['tag1'])
+        assert_equals(model.Session.query(model.PackageExtra).all(), [])
+        # the only member left is for the user created in factories.Group() and
+        # factories.Organization()
+        assert_equals(sorted(
+            [(m.table_name, m.group.name)
+             for m in model.Session.query(model.Member).join(model.Group)]),
+            [('user', 'group1'), ('user', org['name'])])
+
+        # all the object revisions were purged too
+        assert_equals(model.Session.query(model.PackageRevision).all(), [])
+        assert_equals(model.Session.query(model.ResourceRevision).all(), [])
+        assert_equals(model.Session.query(model.PackageTagRevision).all(), [])
+        assert_equals(model.Session.query(model.PackageExtraRevision).all(),
+                      [])
+        # Member is not revisioned
+
+        # No Revision objects were purged or created
+        assert_equals(num_revisions_after - num_revisions_before, 0)
+
+    def test_missing_id_returns_error(self):
+        assert_raises(logic.ValidationError,
+                      helpers.call_action, 'dataset_purge')
+
+    def test_bad_id_returns_404(self):
+        assert_raises(logic.NotFound,
+                      helpers.call_action, 'dataset_purge', id='123')

test-core.ini

@@ -127,7 +127,10 @@ level = INFO
 [logger_sqlalchemy]
 handlers =
 qualname = sqlalchemy.engine
-level = WARN
+level = WARNING
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither.
 
 [handler_console]
 class = StreamHandler