Backporting Master PRs

.gitignore

@@ -8,6 +8,7 @@ MANIFEST
 dist
 .idea/
 .vscode/
+.pytest_cache/
 # supporting stuff pulled in by paver script
 geonode/development.db
 /geoserver/

geonode/base/forms.py

@@ -41,7 +41,7 @@
 from bootstrap3_datetime.widgets import DateTimePicker
 from modeltranslation.forms import TranslationModelForm
 
-from geonode.base.models import TopicCategory, Region, License
+from geonode.base.models import HierarchicalKeyword, TopicCategory, Region, License
 from geonode.people.models import Profile
 from geonode.base.enumerations import ALL_LANGUAGES
 from django.contrib.auth.models import Group
@@ -424,9 +424,18 @@ def unicode_escape(unistr):
             if not isinstance(_k, basestring):
                 for _kk in [x.strip() for x in _k]:
                     _kk = HTMLParser.HTMLParser().unescape(unicode_escape(_kk))
-                    _unsescaped_kwds.append(_kk)
+                    # _hk = HierarchicalKeyword.objects.extra(where=["%s LIKE name||'%%'"], params=[_kk])
+                    _hk = HierarchicalKeyword.objects.filter(name__contains='%s' % _kk.strip())
+                    if _hk and len(_hk) > 0:
+                        _unsescaped_kwds.append(_hk[0])
+                    else:
+                        _unsescaped_kwds.append(_kk)
             else:
-                _unsescaped_kwds.append(_k)
+                _hk = HierarchicalKeyword.objects.filter(name__iexact=_k)
+                if _hk and len(_hk) > 0:
+                    _unsescaped_kwds.append(_hk[0])
+                else:
+                    _unsescaped_kwds.append(_k)
         return _unsescaped_kwds
 
     class Meta:

geonode/base/populate_test_data.py

@@ -264,29 +264,29 @@ def remove_models(obj_ids, type=None):
         remove_models(None, type='document')
 
     if type == 'map':
-        m_ids = obj_ids or [m.id for m in Map.objects.all()]
-        for id in m_ids:
-            try:
+        try:
+            m_ids = obj_ids or [m.id for m in Map.objects.all()]
+            for id in m_ids:
                 m = Map.objects.get(pk=id)
                 m.delete()
-            except:
-                pass
+        except:
+            pass
     elif type == 'layer':
-        l_ids = obj_ids or [l.id for l in Layer.objects.all()]
-        for id in l_ids:
-            try:
+        try:
+            l_ids = obj_ids or [l.id for l in Layer.objects.all()]
+            for id in l_ids:
                 l = Layer.objects.get(pk=id)
                 l.delete()
-            except:
-                pass
+        except:
+            pass
     elif type == 'document':
-        d_ids = obj_ids or [d.id for d in Document.objects.all()]
-        for id in d_ids:
-            try:
+        try:
+            d_ids = obj_ids or [d.id for d in Document.objects.all()]
+            for id in d_ids:
                 d = Document.objects.get(pk=id)
                 d.delete()
-            except:
-                pass
+        except:
+            pass
 
 
 def dump_models(path=None):

geonode/client/templates/admin/themes/custom_theme_html.txt

@@ -35,7 +35,7 @@
 {% endif %}
 
 {{ partners_header|safe }}
-{% if theme.partners %}
+{% if theme.partners_title %}
 <section id="partners">
   <div class="container text-center">
       <h2>{{ theme.partners_title }}</h2>

geonode/geoserver/helpers.py

@@ -902,7 +902,7 @@ def set_styles(layer, gs_catalog):
     if not gs_layer:
         gs_layer = gs_catalog.get_layer(layer.alternate)
 
-    if gs_layer.default_style:
+    if gs_layer and gs_layer.default_style:
         default_style = gs_layer.default_style
     else:
         default_style = gs_catalog.get_style(layer.name, workspace=settings.DEFAULT_WORKSPACE) \
@@ -912,16 +912,18 @@ def set_styles(layer, gs_catalog):
             gs_catalog.save(gs_layer)
         except:
             logger.exception("GeoServer Layer Default Style issues!")
-    layer.default_style = save_style(default_style)
-    # FIXME: This should remove styles that are no longer valid
-    style_set.append(layer.default_style)
+    if default_style:
+        layer.default_style = save_style(default_style)
+        # FIXME: This should remove styles that are no longer valid
+        style_set.append(layer.default_style)
 
-    alt_styles = gs_layer.styles
+    if gs_layer and gs_layer.styles:
+        alt_styles = gs_layer.styles
 
-    for alt_style in alt_styles:
-        style_set.append(save_style(alt_style))
+        for alt_style in alt_styles:
+            style_set.append(save_style(alt_style))
 
-    layer.styles = style_set
+        layer.styles = style_set
 
     # Update default style to database
     to_update = {
@@ -1102,7 +1104,8 @@ def _create_db_featurestore(name, data, overwrite=False, charset="UTF-8", worksp
     """
     cat = gs_catalog
     db = ogc_server_settings.datastore_db
-    dsname = ogc_server_settings.DATASTORE
+    # dsname = ogc_server_settings.DATASTORE
+    dsname = db['NAME']
 
     ds_exists = False
     try:

geonode/geoserver/ows.py

@@ -49,26 +49,29 @@ def _wcs_get_capabilities():
     })
 
 
-def _wcs_link(wcs_url, identifier, mime, srid, bbox):
-    return wcs_url + urllib.urlencode({
+def _wcs_link(wcs_url, identifier, mime, srid=None, bbox=None):
+    wcs_params = {
         'service': 'WCS',
         'request': 'GetCoverage',
         'coverageid': identifier,
         'format': mime,
         'version': '2.0.1',
-        'srs': srid,
-        'bbox': bbox,
-    })
+    }
+    if srid:
+        wcs_params['srs'] = srid
+    if bbox:
+        wcs_params['bbox'] = bbox
+    return wcs_url + urllib.urlencode(wcs_params)
 
 
-def wcs_links(wcs_url, identifier, bbox, srid):
+def wcs_links(wcs_url, identifier, bbox=None, srid=None):
     types = [
         ("x-gzip", _("GZIP"), "application/x-gzip"),
         ("geotiff", _("GeoTIFF"), "image/tiff"),
     ]
     output = []
     for ext, name, mime in types:
-        url = _wcs_link(wcs_url, identifier, mime, srid, bbox)
+        url = _wcs_link(wcs_url, identifier, mime, bbox=bbox, srid=srid)
         output.append((ext, name, mime, url))
     return output
 
@@ -87,30 +90,34 @@ def _wfs_get_capabilities():
     })
 
 
-def _wfs_link(wfs_url, identifier, mime, extra_params):
+def _wfs_link(wfs_url, identifier, mime, extra_params, bbox=None, srid=None):
     params = {
         'service': 'WFS',
         'version': '1.0.0',
         'request': 'GetFeature',
         'typename': identifier,
-        'outputFormat': mime
+        'outputFormat': mime,
     }
+    if srid:
+        params['srs'] = srid
+    if bbox:
+        params['bbox'] = bbox
     params.update(extra_params)
     return wfs_url + urllib.urlencode(params)
 
 
-def wfs_links(wfs_url, identifier):
+def wfs_links(wfs_url, identifier, bbox=None, srid=None):
     types = [
         ("zip", _("Zipped Shapefile"), "SHAPE-ZIP", {'format_options': 'charset:UTF-8'}),
         ("gml", _("GML 2.0"), "gml2", {}),
         ("gml", _("GML 3.1.1"), "text/xml; subtype=gml/3.1.1", {}),
         ("csv", _("CSV"), "csv", {}),
         ("excel", _("Excel"), "excel", {}),
-        ("json", _("GeoJSON"), "json", {'srsName': 'EPSG:4326'})
+        ("json", _("GeoJSON"), "json", {'srsName': srid or 'EPSG:4326'})
     ]
     output = []
     for ext, name, mime, extra_params in types:
-        url = _wfs_link(wfs_url, identifier, mime, extra_params)
+        url = _wfs_link(wfs_url, identifier, mime, extra_params, bbox=bbox, srid=srid)
         output.append((ext, name, mime, url))
     return output
 
@@ -129,17 +136,21 @@ def _wms_get_capabilities():
     })
 
 
-def _wms_link(wms_url, identifier, mime, height, width, srid, bbox):
-    return wms_url + urllib.urlencode({
+def _wms_link(wms_url, identifier, mime, height, width, srid=None, bbox=None):
+    wms_params = {
         'service': 'WMS',
         'request': 'GetMap',
         'layers': identifier,
         'format': mime,
         'height': height,
         'width': width,
-        'srs': srid,
-        'bbox': bbox,
-    })
+    }
+    if srid:
+        wms_params['srs'] = srid
+    if bbox:
+        wms_params['bbox'] = bbox
+
+    return wms_url + urllib.urlencode(wms_params)
 
 
 def wms_links(wms_url, identifier, bbox, srid, height, width):

geonode/geoserver/signals.py

@@ -24,13 +24,14 @@
 
 from urlparse import urlparse, urljoin
 
-from django.utils.translation import ugettext
 from django.conf import settings
+from django.core.urlresolvers import reverse
+from django.utils.translation import ugettext
 from django.forms.models import model_to_dict
 
 
 # use different name to avoid module clash
-from geonode import geoserver as geoserver_app
+from . import BACKEND_PACKAGE
 from geonode import GeoNodeException
 from geonode.decorators import on_ogc_backend
 from geonode.geoserver.ows import wcs_links, wfs_links, wms_links
@@ -76,7 +77,7 @@ def geoserver_pre_save(*args, **kwargs):
     pass
 
 
-@on_ogc_backend(geoserver_app.BACKEND_PACKAGE)
+@on_ogc_backend(BACKEND_PACKAGE)
 def geoserver_post_save(instance, sender, **kwargs):
     from geonode.messaging import producer
     # this is attached to various models, (ResourceBase, Document)
@@ -252,7 +253,7 @@ def geoserver_post_save_local(instance, *args, **kwargs):
                 gs_catalog.save(gs_resource)
 
     if not settings.FREETEXT_KEYWORDS_READONLY:
-        if gs_resource.keywords:
+        if len(instance.keyword_list()) == 0 and gs_resource.keywords:
             for keyword in gs_resource.keywords:
                 if keyword not in instance.keyword_list():
                     instance.keywords.add(keyword)
@@ -311,10 +312,32 @@ def geoserver_post_save_local(instance, *args, **kwargs):
     height = 550
     width = int(height * dataAspect)
 
+    # Parse Layer BBOX and SRID
+    srid = instance.srid if instance.srid else getattr(settings, 'DEFAULT_MAP_CRS', 'EPSG:4326')
+    if srid and instance.bbox_x0:
+        bbox = ','.join(str(x) for x in [instance.bbox_x0, instance.bbox_y0,
+                                         instance.bbox_x1, instance.bbox_y1])
+
+    # Create Raw Data download link
+    path = gs_resource.dom.findall('nativeName')
+    download_url = urljoin(settings.SITEURL,
+                           reverse('download', args=[instance.id]))
+    Link.objects.get_or_create(resource=instance.resourcebase_ptr,
+                               url=download_url,
+                               defaults=dict(extension='zip',
+                                             name='Original Dataset',
+                                             mime='application/octet-stream',
+                                             link_type='original',
+                                             )
+                               )
+
     # Set download links for WMS, WCS or WFS and KML
     links = wms_links(ogc_server_settings.public_url + 'wms?',
-                      instance.alternate.encode('utf-8'), instance.bbox_string,
-                      instance.srid, height, width)
+                      instance.alternate.encode('utf-8'),
+                      bbox,
+                      srid,
+                      height,
+                      width)
 
     for ext, name, mime, wms_url in links:
         Link.objects.get_or_create(resource=instance.resourcebase_ptr,
@@ -328,10 +351,10 @@ def geoserver_post_save_local(instance, *args, **kwargs):
                                    )
 
     if instance.storeType == "dataStore":
-        links = wfs_links(
-            ogc_server_settings.public_url +
-            'wfs?',
-            instance.alternate.encode('utf-8'))
+        links = wfs_links(ogc_server_settings.public_url + 'wfs?',
+                          instance.alternate.encode('utf-8'),
+                          bbox=None,  # bbox filter should be set at runtime otherwise conflicting with CQL
+                          srid=srid)
         for ext, name, mime, wfs_url in links:
             if mime == 'SHAPE-ZIP':
                 name = 'Zipped Shapefile'
@@ -396,8 +419,8 @@ def command_url(command):
     elif instance.storeType == 'coverageStore':
         links = wcs_links(ogc_server_settings.public_url + 'wcs?',
                           instance.alternate.encode('utf-8'),
-                          ','.join(str(x) for x in instance.bbox[0:4]),
-                          instance.srid)
+                          bbox,
+                          srid)
 
     for ext, name, mime, wcs_url in links:
         Link.objects.get_or_create(resource=instance.resourcebase_ptr,

geonode/geoserver/views.py

@@ -285,16 +285,17 @@ def layer_style_manage(request, layername):
             # Save to GeoServer
             cat = gs_catalog
             gs_layer = cat.get_layer(layer.name)
-            gs_layer.default_style = cat.get_style(default_style, workspace=settings.DEFAULT_WORKSPACE) or \
-                cat.get_style(default_style)
-            styles = []
-            for style in selected_styles:
-                styles.append(
-                    cat.get_style(
-                        style,
-                        workspace=settings.DEFAULT_WORKSPACE) or cat.get_style(style))
-            gs_layer.styles = styles
-            cat.save(gs_layer)
+            if gs_layer:
+                gs_layer.default_style = cat.get_style(default_style, workspace=settings.DEFAULT_WORKSPACE) or \
+                    cat.get_style(default_style)
+                styles = []
+                for style in selected_styles:
+                    styles.append(
+                        cat.get_style(
+                            style,
+                            workspace=settings.DEFAULT_WORKSPACE) or cat.get_style(style))
+                gs_layer.styles = styles
+                cat.save(gs_layer)
 
             # Save to Django
             layer = set_styles(layer, cat)

geonode/layers/admin.py

@@ -92,7 +92,7 @@ class LayerFileInline(admin.TabularInline):
 
 class UploadSessionAdmin(admin.ModelAdmin):
     model = UploadSession
-    list_display = ('date', 'user', 'processed')
+    list_display = ('resource', 'date', 'user', 'processed')
     inlines = [LayerFileInline]
 
 

geonode/layers/migrations/0031_uploadsession_resource.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2018-04-24 16:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0033_auto_20180330_0951'),
+        ('layers', '0030_auto_20180228_0812'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='uploadsession',
+            name='resource',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='base.ResourceBase'),
+        ),
+    ]