[Closes #3662] GNIP: Improvements to GeoNode Layers download links

geonode/geoserver/ows.py

@@ -49,26 +49,29 @@ def _wcs_get_capabilities():
     })
 
 
-def _wcs_link(wcs_url, identifier, mime, srid, bbox):
-    return wcs_url + urllib.urlencode({
+def _wcs_link(wcs_url, identifier, mime, srid=None, bbox=None):
+    wcs_params = {
         'service': 'WCS',
         'request': 'GetCoverage',
         'coverageid': identifier,
         'format': mime,
         'version': '2.0.1',
-        'srs': srid,
-        'bbox': bbox,
-    })
+    }
+    if srid:
+        wcs_params['srs'] = srid
+    if bbox:
+        wcs_params['bbox'] = bbox
+    return wcs_url + urllib.urlencode(wcs_params)
 
 
-def wcs_links(wcs_url, identifier, bbox, srid):
+def wcs_links(wcs_url, identifier, bbox=None, srid=None):
     types = [
         ("x-gzip", _("GZIP"), "application/x-gzip"),
         ("geotiff", _("GeoTIFF"), "image/tiff"),
     ]
     output = []
     for ext, name, mime in types:
-        url = _wcs_link(wcs_url, identifier, mime, srid, bbox)
+        url = _wcs_link(wcs_url, identifier, mime, bbox=bbox, srid=srid)
         output.append((ext, name, mime, url))
     return output
 
@@ -87,30 +90,34 @@ def _wfs_get_capabilities():
     })
 
 
-def _wfs_link(wfs_url, identifier, mime, extra_params):
+def _wfs_link(wfs_url, identifier, mime, extra_params, bbox=None, srid=None):
     params = {
         'service': 'WFS',
         'version': '1.0.0',
         'request': 'GetFeature',
         'typename': identifier,
-        'outputFormat': mime
+        'outputFormat': mime,
     }
+    if srid:
+        params['srs'] = srid
+    if bbox:
+        params['bbox'] = bbox
     params.update(extra_params)
     return wfs_url + urllib.urlencode(params)
 
 
-def wfs_links(wfs_url, identifier):
+def wfs_links(wfs_url, identifier, bbox=None, srid=None):
     types = [
         ("zip", _("Zipped Shapefile"), "SHAPE-ZIP", {'format_options': 'charset:UTF-8'}),
         ("gml", _("GML 2.0"), "gml2", {}),
         ("gml", _("GML 3.1.1"), "text/xml; subtype=gml/3.1.1", {}),
         ("csv", _("CSV"), "csv", {}),
         ("excel", _("Excel"), "excel", {}),
-        ("json", _("GeoJSON"), "json", {'srsName': 'EPSG:4326'})
+        ("json", _("GeoJSON"), "json", {'srsName': srid or 'EPSG:4326'})
     ]
     output = []
     for ext, name, mime, extra_params in types:
-        url = _wfs_link(wfs_url, identifier, mime, extra_params)
+        url = _wfs_link(wfs_url, identifier, mime, extra_params, bbox=bbox, srid=srid)
         output.append((ext, name, mime, url))
     return output
 
@@ -129,17 +136,21 @@ def _wms_get_capabilities():
     })
 
 
-def _wms_link(wms_url, identifier, mime, height, width, srid, bbox):
-    return wms_url + urllib.urlencode({
+def _wms_link(wms_url, identifier, mime, height, width, srid=None, bbox=None):
+    wms_params = {
         'service': 'WMS',
         'request': 'GetMap',
         'layers': identifier,
         'format': mime,
         'height': height,
         'width': width,
-        'srs': srid,
-        'bbox': bbox,
-    })
+    }
+    if srid:
+        wms_params['srs'] = srid
+    if bbox:
+        wms_params['bbox'] = bbox
+
+    return wms_url + urllib.urlencode(wms_params)
 
 
 def wms_links(wms_url, identifier, bbox, srid, height, width):

geonode/geoserver/signals.py

@@ -24,13 +24,14 @@
 
 from urlparse import urlparse, urljoin
 
-from django.utils.translation import ugettext
 from django.conf import settings
+from django.core.urlresolvers import reverse
+from django.utils.translation import ugettext
 from django.forms.models import model_to_dict
 
 
 # use different name to avoid module clash
-from geonode import geoserver as geoserver_app
+from . import BACKEND_PACKAGE
 from geonode import GeoNodeException
 from geonode.decorators import on_ogc_backend
 from geonode.geoserver.ows import wcs_links, wfs_links, wms_links
@@ -76,7 +77,7 @@ def geoserver_pre_save(*args, **kwargs):
     pass
 
 
-@on_ogc_backend(geoserver_app.BACKEND_PACKAGE)
+@on_ogc_backend(BACKEND_PACKAGE)
 def geoserver_post_save(instance, sender, **kwargs):
     from geonode.messaging import producer
     # this is attached to various models, (ResourceBase, Document)
@@ -311,10 +312,32 @@ def geoserver_post_save_local(instance, *args, **kwargs):
     height = 550
     width = int(height * dataAspect)
 
+    # Parse Layer BBOX and SRID
+    srid = instance.srid if instance.srid else getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:4326')
+    if srid and instance.bbox_x0:
+        bbox = ','.join(str(x) for x in [instance.bbox_x0, instance.bbox_y0,
+                                         instance.bbox_x1, instance.bbox_y1])
+
+    # Create Raw Data download link
+    path = gs_resource.dom.findall('nativeName')
+    download_url = urljoin(settings.SITEURL,
+                           reverse('download', args=[instance.id]))
+    Link.objects.get_or_create(resource=instance.resourcebase_ptr,
+                               url=download_url,
+                               defaults=dict(extension='zip',
+                                             name='Original Dataset',
+                                             mime='application/octet-stream',
+                                             link_type='original',
+                                             )
+                               )
+
     # Set download links for WMS, WCS or WFS and KML
     links = wms_links(ogc_server_settings.public_url + 'wms?',
-                      instance.alternate.encode('utf-8'), instance.bbox_string,
-                      instance.srid, height, width)
+                      instance.alternate.encode('utf-8'),
+                      bbox,
+                      srid,
+                      height,
+                      width)
 
     for ext, name, mime, wms_url in links:
         Link.objects.get_or_create(resource=instance.resourcebase_ptr,
@@ -328,10 +351,10 @@ def geoserver_post_save_local(instance, *args, **kwargs):
                                    )
 
     if instance.storeType == "dataStore":
-        links = wfs_links(
-            ogc_server_settings.public_url +
-            'wfs?',
-            instance.alternate.encode('utf-8'))
+        links = wfs_links(ogc_server_settings.public_url + 'wfs?',
+                          instance.alternate.encode('utf-8'),
+                          bbox=None,  # bbox filter should be set at runtime otherwise conflicting with CQL
+                          srid=srid)
         for ext, name, mime, wfs_url in links:
             if mime == 'SHAPE-ZIP':
                 name = 'Zipped Shapefile'
@@ -396,8 +419,8 @@ def command_url(command):
     elif instance.storeType == 'coverageStore':
         links = wcs_links(ogc_server_settings.public_url + 'wcs?',
                           instance.alternate.encode('utf-8'),
-                          ','.join(str(x) for x in instance.bbox[0:4]),
-                          instance.srid)
+                          bbox,
+                          srid)
 
     for ext, name, mime, wcs_url in links:
         Link.objects.get_or_create(resource=instance.resourcebase_ptr,

geonode/layers/admin.py

@@ -92,7 +92,7 @@ class LayerFileInline(admin.TabularInline):
 
 class UploadSessionAdmin(admin.ModelAdmin):
     model = UploadSession
-    list_display = ('date', 'user', 'processed')
+    list_display = ('resource', 'date', 'user', 'processed')
     inlines = [LayerFileInline]
 
 

geonode/layers/migrations/0031_uploadsession_resource.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2018-04-24 16:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0033_auto_20180330_0951'),
+        ('layers', '0030_auto_20180228_0812'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='uploadsession',
+            name='resource',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase'),
+        ),
+    ]

geonode/layers/migrations/0032_auto_20180424_1638.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2018-04-24 16:38
+from __future__ import unicode_literals
+
+import django.core.files.storage
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layers', '0031_uploadsession_resource'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='layerfile',
+            name='file',
+            field=models.FileField(max_length=255, storage=django.core.files.storage.FileSystemStorage(base_url=b'/uploaded/'), upload_to=b'layers/%Y/%m/%d'),
+        ),
+    ]

geonode/layers/models.py

@@ -329,6 +329,7 @@ class UploadSession(models.Model):
 
     """Helper class to keep track of uploads.
     """
+    resource = models.ForeignKey(ResourceBase, blank=True, null=True)
     date = models.DateTimeField(auto_now=True)
     user = models.ForeignKey(settings.AUTH_USER_MODEL)
     processed = models.BooleanField(default=False)
@@ -339,6 +340,9 @@ class UploadSession(models.Model):
     def successful(self):
         return self.processed and self.errors is None
 
+    def __str__(self):
+        return u'%s' % self.resource or self.date
+
 
 class LayerFile(models.Model):
 
@@ -348,7 +352,7 @@ class LayerFile(models.Model):
     name = models.CharField(max_length=255)
     base = models.BooleanField(default=False)
     file = models.FileField(
-        upload_to='layers',
+        upload_to='layers/%Y/%m/%d',
         storage=FileSystemStorage(
             base_url=settings.LOCAL_MEDIA_URL),
         max_length=255)
@@ -605,18 +609,14 @@ def post_delete_layer(instance, sender, **kwargs):
         return
 
     from geonode.maps.models import MapLayer
-    if instance.alternate:
-        logger.debug(
-            "Going to delete associated maplayers for [%s]",
-            instance.alternate.encode('utf-8'))
-        MapLayer.objects.filter(
-            name=instance.alternate,
-            ows_url=instance.ows_url).delete()
+    logger.debug(
+        "Going to delete associated maplayers for [%s]", instance.name)
+    MapLayer.objects.filter(
+        name=instance.alternate,
+        ows_url=instance.ows_url).delete()
 
-    if instance.alternate:
-        logger.debug(
-            "Going to delete the default style for [%s]",
-            instance.alternate.encode('utf-8'))
+    logger.debug(
+        "Going to delete the default style for [%s]", instance.name)
 
     if instance.default_style and Layer.objects.filter(
             default_style__id=instance.default_style.id).count() == 0:
@@ -626,6 +626,7 @@ def post_delete_layer(instance, sender, **kwargs):
         if instance.upload_session:
             for lf in instance.upload_session.layerfile_set.all():
                 lf.file.delete()
+            instance.upload_session.delete()
     except UploadSession.DoesNotExist:
         pass
 

geonode/layers/utils.py

@@ -570,9 +570,14 @@ def file_upload(filename,
     # process the layer again after that by
     # doing a layer.save()
     if not created and overwrite:
-        if layer.upload_session:
-            layer.upload_session.layerfile_set.all().delete()
         if upload_session:
+            if layer.upload_session:
+                layer.upload_session.layerfile_set.all().delete()
+                layer.upload_session.delete()
+
+            upload_session.resource = layer
+            upload_session.save()
+
             layer.upload_session = upload_session
 
         # update with new information

geonode/proxy/views.py

@@ -261,23 +261,24 @@ def download(request, resourceid, sender=Layer):
             # Let's check for associated SLD files (if any)
             try:
                 for s in instance.styles.all():
-                    sld_file = os.path.join(target_folder, "".join([s.name, ".sld"]))
-                    sld_file = open(sld_file, "w")
+                    sld_file_path = os.path.join(target_folder, "".join([s.name, ".sld"]))
+                    sld_file = open(sld_file_path, "w")
                     sld_file.write(s.sld_body.strip())
                     sld_file.close()
 
-                    sld_file = open(sld_file, "w")
                     try:
+                        sld_file = open(sld_file_path, "r")
                         response = requests.get(s.sld_url, timeout=TIMEOUT)
                         sld_remote_content = response.text
-                        sld_file = os.path.join(target_folder, "".join([s.name, "_remote.sld"]))
+                        sld_file_path = os.path.join(target_folder, "".join([s.name, "_remote.sld"]))
+                        sld_file = open(sld_file_path, "w")
                         sld_file.write(sld_remote_content.strip())
+                        sld_file.close()
                     except:
                         traceback.print_exc()
                         tb = traceback.format_exc()
                         logger.debug(tb)
-                    finally:
-                        sld_file.close()
+
             except:
                 traceback.print_exc()
                 tb = traceback.format_exc()
@@ -293,7 +294,11 @@ def download(request, resourceid, sender=Layer):
                 for link in links:
                     link_name = custom_slugify(link.name)
                     link_file = os.path.join(target_md_folder, "".join([link_name, ".%s" % link.extension]))
-                    if link.link_type in ('metadata', 'data', 'image'):
+                    if link.link_type in ('data'):
+                        # Skipping 'data' download links
+                        continue
+                    elif link.link_type in ('metadata', 'image'):
+                        # Dumping metadata files and images
                         link_file = open(link_file, "wb")
                         try:
                             response = requests.get(link.url, stream=True, timeout=TIMEOUT)
@@ -306,6 +311,7 @@ def download(request, resourceid, sender=Layer):
                         finally:
                             link_file.close()
                     elif link.link_type.startswith('OGC'):
+                        # Dumping OGC/OWS links
                         link_file = open(link_file, "w")
                         link_file.write(link.url.strip())
                         link_file.close()