GeoNode Cleanup and Test Coverage

geonode/documents/tasks.py

@@ -95,5 +95,5 @@ def delete_orphaned_document_files(self):
 
 @shared_task(bind=True, queue='cleanup')
 def delete_orphaned_thumbnails(self):
-    from geonode.documents.utils import delete_orphaned_thumbs
+    from geonode.base.utils import delete_orphaned_thumbs
     delete_orphaned_thumbs()

geonode/documents/tests.py

@@ -102,13 +102,13 @@ def test_create_document_with_rel(self):
 
         m = Map.objects.all()[0]
         ctype = ContentType.objects.get_for_model(m)
-        _l = DocumentResourceLink.objects.create(
+        _d = DocumentResourceLink.objects.create(
             document_id=c.id,
             content_type=ctype,
             object_id=m.id)
 
         self.assertEquals(Document.objects.get(pk=c.id).title, 'theimg')
-        self.assertEquals(DocumentResourceLink.objects.get(pk=_l.id).object_id,
+        self.assertEquals(DocumentResourceLink.objects.get(pk=_d.id).object_id,
                           m.id)
 
     def test_create_document_url(self):
@@ -264,6 +264,17 @@ def test_document_isuploaded(self):
             follow=True)
         self.assertEquals(response.status_code, 200)
 
+        _d = Document.objects.get(title='uploaded_document')
+        _d.delete()
+
+        from geonode.documents.utils import delete_orphaned_document_files
+        delete_orphaned_document_files()
+
+        from geonode.base.utils import delete_orphaned_thumbs
+        delete_orphaned_thumbs()
+
+        self.assertFalse(os.path.isfile(f))
+
     # Permissions Tests
 
     def test_set_document_permissions(self):
@@ -478,10 +489,10 @@ def test_moderated_upload(self):
                 resp = self.client.post(document_upload_url, data=data)
             self.assertEqual(resp.status_code, 302)
             dname = 'document title'
-            _l = Document.objects.get(title=dname)
+            _d = Document.objects.get(title=dname)
 
-            self.assertTrue(_l.is_published)
-            _l.delete()
+            self.assertTrue(_d.is_published)
+            _d.delete()
 
         with self.settings(ADMIN_MODERATE_UPLOADS=True):
             document_upload_url = reverse('document_upload')
@@ -499,9 +510,9 @@ def test_moderated_upload(self):
                 resp = self.client.post(document_upload_url, data=data)
             self.assertEqual(resp.status_code, 302)
             dname = 'document title'
-            _l = Document.objects.get(title=dname)
+            _d = Document.objects.get(title=dname)
 
-            self.assertFalse(_l.is_published)
+            self.assertFalse(_d.is_published)
 
 
 class DocumentNotificationsTestCase(NotificationsTestsHelper):
@@ -520,17 +531,17 @@ def setUp(self):
     def testDocumentNotifications(self):
         with self.settings(PINAX_NOTIFICATIONS_QUEUE_ALL=True):
             self.clear_notifications_queue()
-            _l = Document.objects.create(title='test notifications', owner=self.u)
+            _d = Document.objects.create(title='test notifications', owner=self.u)
             self.assertTrue(self.check_notification_out('document_created', self.u))
-            _l.title = 'test notifications 2'
-            _l.save()
+            _d.title = 'test notifications 2'
+            _d.save()
             self.assertTrue(self.check_notification_out('document_updated', self.u))
 
             from dialogos.models import Comment
-            lct = ContentType.objects.get_for_model(_l)
+            lct = ContentType.objects.get_for_model(_d)
             comment = Comment(author=self.u, name=self.u.username,
-                              content_type=lct, object_id=_l.id,
-                              content_object=_l, comment='test comment')
+                              content_type=lct, object_id=_d.id,
+                              content_object=_d, comment='test comment')
             comment.save()
 
             self.assertTrue(self.check_notification_out('document_comment', self.u))
@@ -579,12 +590,12 @@ def test_create_document_with_links(self):
 
         for resource in resources:
             ct = ContentType.objects.get_for_model(resource)
-            _l = DocumentResourceLink.objects.get(
+            _d = DocumentResourceLink.objects.get(
                 document_id=d.id,
                 content_type=ct.id,
                 object_id=resource.id
             )
-            self.assertEquals(_l.object_id, resource.id)
+            self.assertEquals(_d.object_id, resource.id)
 
         # update document links
 
@@ -597,12 +608,12 @@ def test_create_document_with_links(self):
 
         for resource in layers:
             ct = ContentType.objects.get_for_model(resource)
-            _l = DocumentResourceLink.objects.get(
+            _d = DocumentResourceLink.objects.get(
                 document_id=d.id,
                 content_type=ct.id,
                 object_id=resource.id
             )
-            self.assertEquals(_l.object_id, resource.id)
+            self.assertEquals(_d.object_id, resource.id)
 
         for resource in maps:
             ct = ContentType.objects.get_for_model(resource)

geonode/geoserver/urls.py

@@ -83,7 +83,5 @@
     url(r'^acls/?$', views.layer_acls, name='layer_acls'),
     url(r'^resolve_user/?$', views.resolve_user,
         name='layer_resolve_user'),
-    url(r'^download$', views.layer_batch_download,
-        name='layer_batch_download'),
     url(r'^online/?$', views.server_online, name='server_online'),
 ]

geonode/geoserver/views.py

@@ -568,60 +568,6 @@ def _response_callback(**kwargs):
         content_type=content_type)
 
 
-def layer_batch_download(request):
-    """
-    batch download a set of layers
-
-    POST - begin download
-    GET?id=<download_id> monitor status
-    """
-
-    from geonode.utils import http_client
-    # currently this just piggy-backs on the map download backend
-    # by specifying an ad hoc map that contains all layers requested
-    # for download. assumes all layers are hosted locally.
-    # status monitoring is handled slightly differently.
-
-    if request.method == 'POST':
-        layers = request.POST.getlist("layer")
-        layers = Layer.objects.filter(alternate__in=list(layers))
-
-        def layer_son(layer):
-            return {
-                "name": layer.alternate,
-                "service": layer.service_type,
-                "metadataURL": "",
-                "serviceURL": ""
-            }
-
-        readme = """This data is provided by GeoNode.\n\nContents:"""
-
-        def list_item(lyr):
-            return "%s - %s.*" % (lyr.title, lyr.name)
-
-        readme = "\n".join([readme] + [list_item(l) for l in layers])
-
-        fake_map = {
-            "map": {"readme": readme},
-            "layers": [layer_son(lyr) for lyr in layers]
-        }
-
-        url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
-        req, content = http_client.post(url, data=json.dumps(fake_map))
-        return HttpResponse(content, status=req.status_code)
-
-    if request.method == 'GET':
-        # essentially, this just proxies back to geoserver
-        download_id = request.GET.get('id', None)
-        if download_id is None:
-            return HttpResponse(status=404)
-
-        url = "%srest/process/batchDownload/status/%s" % (
-            ogc_server_settings.LOCATION, download_id)
-        req, content = http_client.get(url)
-        return HttpResponse(content, status=req.status_code)
-
-
 def resolve_user(request):
     user = None
     geoserver = False

geonode/layers/urls.py

@@ -66,20 +66,14 @@
         views.layer_feature_catalogue, name='layer_feature_catalogue'),
     url(r'^metadata/batch/(?P<ids>[^/]*)/$',
         views.layer_batch_metadata, name='layer_batch_metadata'),
-
-    # url(r'^api/batch_permissions/?$', 'batch_permissions',
-    #    name='batch_permssions'),
-    # url(r'^api/batch_delete/?$', 'batch_delete', name='batch_delete'),
 ]
 
 # -- Deprecated url routes for Geoserver authentication -- remove after GeoNode 2.1
 # -- Use /gs/acls, gs/resolve_user/, gs/download instead
 if check_ogc_backend(geoserver.BACKEND_PACKAGE):
-    from geonode.geoserver.views import layer_acls, resolve_user, layer_batch_download
+    from geonode.geoserver.views import layer_acls, resolve_user
     urlpatterns = [  # 'geonode.geoserver.views',
         url(r'^acls/?$', layer_acls, name='layer_acls_dep'),
         url(r'^resolve_user/?$', resolve_user,
             name='layer_resolve_user_dep'),
-        url(r'^download$', layer_batch_download,
-            name='layer_batch_download_dep'),
     ] + urlpatterns

geonode/maps/urls.py

@@ -72,7 +72,6 @@
     url(r'^(?P<mapid>[^/]+)/view$', existing_map_view, name='map_view'),
     url(r'^(?P<mapid>[^/]+)/edit$', map_edit, name='map_edit'),
     url(r'^(?P<mapid>[^/]+)/data$', map_json, name='map_json'),
-    url(r'^(?P<mapid>[^/]+)/download$', views.map_download, name='map_download'),
     url(r'^(?P<mapid>[^/]+)/wmc$', views.map_wmc, name='map_wmc'),
     url(r'^(?P<mapid>[^/]+)/wms$', views.map_wms, name='map_wms'),
     url(r'^(?P<mapid>[^/]+)/remove$', views.map_remove, name='map_remove'),
@@ -90,7 +89,6 @@
     url(r'^(?P<mapid>[^/]+)/(?P<snapshot>[A-Za-z0-9_\-]+)/data$',
         map_json,
         name='map_json'),
-    url(r'^check/$', views.map_download_check, name='map_download_check'),
     url(r'^embed/$', views.map_embed, name='map_embed'),
     url(r'^metadata/batch/(?P<ids>[^/]*)/$', views.map_batch_metadata, name='map_batch_metadata'),
     url(r'^(?P<mapid>[^/]*)/metadata_detail$',
@@ -99,12 +97,14 @@
     url(r'^(?P<layername>[^/]*)/attributes',
         views.maplayer_attributes,
         name='maplayer_attributes'),
-    # url(r'^change-poc/(?P<ids>\w+)$', views.change_poc, name='maps_change_poc'),
 ]
 
 if check_ogc_backend(qgis_server.BACKEND_PACKAGE):
     # Add QLR url specific for QGIS Server
     urlpatterns += [
+        url(r'^(?P<mapid>[^/]+)/download$',
+            views.map_download,
+            name='map_download'),
         url(r'^(?P<mapid>[^/]+)/qlr$',
             map_download_qlr,
             name='map_download_qlr'),

geonode/maps/views.py

@@ -1042,10 +1042,7 @@ def perm_filter(layer):
                 j_layers.remove(j_layer)
         mapJson = json.dumps(j_map)
 
-        if check_ogc_backend(geoserver.BACKEND_PACKAGE):
-            # TODO the url needs to be verified on geoserver
-            url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
-        elif check_ogc_backend(qgis_server.BACKEND_PACKAGE):
+        if check_ogc_backend(qgis_server.BACKEND_PACKAGE):
             url = urljoin(settings.SITEURL,
                           reverse("qgis_server:download-map", kwargs={'mapid': mapid}))
             # qgis-server backend stop here, continue on qgis_server/views.py
@@ -1095,31 +1092,6 @@ def perm_filter(layer):
     })
 
 
-def map_download_check(request):
-    """
-    this is an endpoint for monitoring map downloads
-    """
-    try:
-        layer = request.session["map_status"]
-        if isinstance(layer, dict):
-            url = "%srest/process/batchDownload/status/%s" % (
-                ogc_server_settings.LOCATION, layer["id"])
-            resp, content = http_client.request(url, 'GET')
-            status = resp.status_code
-            if resp.status_code == 400:
-                return HttpResponse(
-                    content="Something went wrong",
-                    status=status)
-        else:
-            content = "Something Went wrong"
-            status = 400
-    except ValueError:
-        # TODO: Is there any useful context we could include in this log?
-        logger.warning(
-            "User tried to check status, but has no download in progress.")
-    return HttpResponse(content=content, status=status)
-
-
 def map_wmc(request, mapid, template="maps/wmc.xml"):
     """Serialize an OGC Web Map Context Document (WMC) 1.1"""
     map_obj = _resolve_map(

geonode/security/tests.py

@@ -35,13 +35,10 @@
 from django.contrib.auth import get_user_model
 from guardian.shortcuts import get_anonymous_user, assign_perm, remove_perm
 from geonode import geoserver
-from geonode.settings import on_travis
 from geonode.base.populate_test_data import all_public
-from geonode.maps.tests_populate_maplayers import create_maplayers
 from geonode.people.models import Profile
 from geonode.people.utils import get_valid_user
 from geonode.layers.models import Layer
-from geonode.maps.models import Map
 from geonode.groups.models import Group
 from geonode.utils import check_ogc_backend
 from geonode.tests.utils import check_layer
@@ -1182,36 +1179,6 @@ def test_anonymus_permissions(self):
             response = self.client.get(reverse('layer_style_manage', args=(layer.alternate,)))
             self.assertEquals(response.status_code, 302)
 
-    def test_map_download(self):
-        """Test the correct permissions on layers on map download"""
-        if not on_travis:
-            create_maplayers()
-            # Get a Map
-            the_map = Map.objects.get(title='GeoNode Default Map')
-
-            # Get a MapLayer and set the parameters as it is local and not a background
-            # and leave it alone in the map
-            map_layer = the_map.layer_set.get(name='geonode:CA')
-            map_layer.local = True
-            map_layer.group = 'overlay'
-            map_layer.save()
-            the_map.layer_set.all().delete()
-            the_map.layer_set.add(map_layer)
-
-            # Get the Layer and set the permissions for bobby to it and the map
-            bobby = Profile.objects.get(username='bobby')
-            the_layer = Layer.objects.get(alternate='geonode:CA')
-            remove_perm('download_resourcebase', bobby, the_layer.get_self_resource())
-            remove_perm('download_resourcebase', Group.objects.get(name='anonymous'),
-                        the_layer.get_self_resource())
-            assign_perm('view_resourcebase', bobby, the_layer.get_self_resource())
-            assign_perm('download_resourcebase', bobby, the_map.get_self_resource())
-
-            self.client.login(username='bobby', password='bob')
-
-            response = self.client.get(reverse('map_download', args=(the_map.id,)))
-            self.assertTrue('Could not find downloadable layers for this map' in response.content)
-
 
 class GisBackendSignalsTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport):
 

pavement.py

@@ -934,8 +934,8 @@ def run_tests(options):
     """
     if options.get('coverage'):
         prefix = 'coverage run --branch --source=geonode \
-            --omit="*/management/*,*/__init__*,*/views*,*/signals*,*/tasks*,*/test*,*/wsgi*,*/middleware*,\
-                */migrations*,*/context_processors*,geonode/qgis_server/*,geonode/contrib/*,geonode/upload/*"'
+            --omit="*/management/*,*/__init__*,*/views*,*/signals*,*/tasks*,*/test*,*/wsgi*,*/middleware*,*/search_indexes*,\
+                */migrations*,*/context_processors*,geonode/qgis_server/*,geonode/monitoring/*,geonode/upload/*"'
     else:
         prefix = 'python'
     local = options.get('local', 'false')  # travis uses default to false