- Delayed GeoFence Security Rules Invalidation

.gitignore

@@ -81,3 +81,5 @@ celerybeat-schedule.*
 geonode\.tests\.bdd\.e2e\.test_login/
 
 *.log
+
+/celerybeat.pid

geonode/api/api.py

@@ -37,6 +37,7 @@
 from tastypie.exceptions import BadRequest
 
 from geonode import qgis_server, geoserver
+from geonode.api.paginator import CrossSiteXHRPaginator
 from geonode.api.authorization import GeoNodeStyleAuthorization
 from geonode.qgis_server.models import QGISServerStyle
 from guardian.shortcuts import get_objects_for_user
@@ -520,6 +521,7 @@ class QGISStyleResource(ModelResource):
     type = fields.CharField(attribute='type')
 
     class Meta:
+        paginator_class = CrossSiteXHRPaginator
         queryset = QGISServerStyle.objects.all()
         resource_name = 'styles'
         detail_uri_name = 'id'
@@ -699,6 +701,7 @@ class GeoserverStyleResource(ModelResource):
     type = fields.CharField(attribute='type')
 
     class Meta:
+        paginator_class = CrossSiteXHRPaginator
         queryset = Style.objects.all()
         resource_name = 'styles'
         detail_uri_name = 'id'

geonode/api/paginator.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#########################################################################
+#
+# Copyright (C) 2018 OSGeo
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#########################################################################
+
+from django.conf import settings
+from tastypie.exceptions import BadRequest
+from tastypie.paginator import Paginator
+
+
+class CrossSiteXHRPaginator(Paginator):
+
+    def get_limit(self):
+        """
+        Determines the proper maximum number of results to return.
+
+        In order of importance, it will use:
+
+            * The user-requested ``limit`` from the GET parameters, if specified.
+            * The object-level ``limit`` if specified.
+            * ``settings.API_LIMIT_PER_PAGE`` if specified.
+
+        Default is 20 per page.
+        """
+
+        limit = self.request_data.get('limit', self.limit)
+        if limit is None:
+            limit = getattr(settings, 'API_LIMIT_PER_PAGE', 20)
+
+        try:
+            limit = int(limit)
+        except ValueError:
+            raise BadRequest("Invalid limit provided. Please provide a positive integer.")
+
+        if limit < 0:
+            raise BadRequest("Invalid limit provided. Please provide a positive integer >= 0.")
+
+        if self.max_limit and (not limit or limit > self.max_limit):
+            # If it's more than the max, we're only going to return the max.
+            # This is to prevent excessive DB (or other) load.
+            return self.max_limit
+
+        return limit
+
+    def get_offset(self):
+        """
+        Determines the proper starting offset of results to return.
+
+        It attempts to use the user-provided ``offset`` from the GET parameters,
+        if specified. Otherwise, it falls back to the object-level ``offset``.
+
+        Default is 0.
+        """
+        offset = self.offset
+
+        if 'offset' in self.request_data:
+            offset = self.request_data['offset']
+
+        try:
+            offset = int(offset)
+        except ValueError:
+            raise BadRequest("Invalid offset provided. Please provide an integer.")
+
+        if offset < 0:
+            raise BadRequest("Invalid offset provided. Please provide a positive integer >= 0.")
+
+        return offset

geonode/api/resourcebase_api.py

@@ -58,10 +58,14 @@
 
 from .authorization import GeoNodeAuthorization, GeonodeApiKeyAuthentication
 
-from .api import TagResource, RegionResource, OwnersResource
-from .api import ThesaurusKeywordResource
-from .api import TopicCategoryResource, GroupResource
-from .api import FILTER_TYPES
+from .api import (TagResource,
+                  RegionResource,
+                  OwnersResource,
+                  ThesaurusKeywordResource,
+                  TopicCategoryResource,
+                  GroupResource,
+                  FILTER_TYPES)
+from .paginator import CrossSiteXHRPaginator
 
 if settings.HAYSTACK_SEARCH:
     from haystack.query import SearchQuerySet  # noqa
@@ -145,6 +149,7 @@ class CommonModelApi(ModelResource):
         'has_time',
         'is_approved',
         'is_published',
+        'dirty_state',
     ]
 
     def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs):
@@ -650,6 +655,7 @@ class ResourceBaseResource(CommonModelApi):
     """ResourceBase api"""
 
     class Meta(CommonMetaApi):
+        paginator_class = CrossSiteXHRPaginator
         queryset = ResourceBase.objects.polymorphic_queryset() \
             .distinct().order_by('-date')
         resource_name = 'base'
@@ -662,6 +668,7 @@ class FeaturedResourceBaseResource(CommonModelApi):
     """Only the featured resourcebases"""
 
     class Meta(CommonMetaApi):
+        paginator_class = CrossSiteXHRPaginator
         queryset = ResourceBase.objects.filter(featured=True).order_by('-date')
         resource_name = 'featured'
         authentication = MultiAuthentication(SessionAuthentication(), GeonodeApiKeyAuthentication())
@@ -864,6 +871,7 @@ def patch_detail(self, request, **kwargs):
     VALUES.append('typename')
 
     class Meta(CommonMetaApi):
+        paginator_class = CrossSiteXHRPaginator
         queryset = Layer.objects.distinct().order_by('-date')
         resource_name = 'layers'
         detail_uri_name = 'id'
@@ -941,6 +949,7 @@ def format_objects(self, objects):
         return formatted_objects
 
     class Meta(CommonMetaApi):
+        paginator_class = CrossSiteXHRPaginator
         queryset = Map.objects.distinct().order_by('-date')
         resource_name = 'maps'
         authentication = MultiAuthentication(SessionAuthentication(), GeonodeApiKeyAuthentication())
@@ -985,6 +994,7 @@ def format_objects(self, objects):
         return formatted_objects
 
     class Meta(CommonMetaApi):
+        paginator_class = CrossSiteXHRPaginator
         filtering = CommonMetaApi.filtering
         filtering.update({'doc_type': ALL})
         queryset = Document.objects.distinct().order_by('-date')

geonode/base/migrations/0035_resourcebase_dirty_state.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2018-10-02 15:42
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0034_auto_20180606_1543'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='resourcebase',
+            name='dirty_state',
+            field=models.BooleanField(default=False, help_text='Security Rules Are Not Synched with GeoServer!', verbose_name='Dirty State'),
+        ),
+    ]

geonode/base/models.py

@@ -762,6 +762,12 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase):
     detail_url = models.CharField(max_length=255, null=True, blank=True)
     rating = models.IntegerField(default=0, null=True, blank=True)
 
+    # fields controlling security state
+    dirty_state = models.BooleanField(
+        _("Dirty State"),
+        default=False,
+        help_text=_('Security Rules Are Not Synched with GeoServer!'))
+
     def __unicode__(self):
         return self.title
 
@@ -921,6 +927,16 @@ def spatial_representation_type_string(self):
             else:
                 return None
 
+    def set_dirty_state(self):
+        if not self.dirty_state:
+            self.dirty_state = True
+            self.save()
+
+    def clear_dirty_state(self):
+        if self.dirty_state:
+            self.dirty_state = False
+            self.save()
+
     @property
     def keyword_csv(self):
         keywords_qs = self.get_real_instance().keywords.all()

geonode/base/templates/base/_resourcebase_snippet.html

@@ -47,8 +47,9 @@ <h4>
             <span ng-if="item.online == false"><i class="fa fa-power-off text-danger"></i> {% trans "Service is" %} {% trans "offline" %}</span>
             {% verbatim %}
           </em>
-          <div class="alert alert-warning" ng-if="item.is_approved == false">PENDING APPROVAL</div>
-          <div class="alert alert-danger" ng-if="item.is_approved == true && item.is_published == false">UNPUBLISHED</div>
+          <div class="alert alert-danger" ng-if="item.dirty_state == true">SECURITY NOT YET SYNCHRONIZED</div>
+          <div class="alert alert-warning" ng-if="item.dirty_state == false && item.is_approved == false">PENDING APPROVAL</div>
+          <div class="alert alert-danger" ng-if="item.dirty_state == false && item.is_approved == true && item.is_published == false">UNPUBLISHED</div>
           <p class="abstract">{{ item.abstract | limitTo: 300 }}{{ item.abstract.length  > 300 ? '...' : ''}}</p>
           <div class="row">
             <div class="col-lg-12 item-items">

geonode/catalogue/backends/pycsw_local.py

@@ -29,8 +29,10 @@
 from shapely.geometry.base import ReadingError
 
 true_value = 'true'
+false_value = 'false'
 if settings.DATABASES['default']['ENGINE'].endswith(('sqlite', 'sqlite3', 'spatialite',)):
     true_value = '1'
+    false_value = '0'
 
 # pycsw settings that the user shouldn't have to worry about
 CONFIGURATION = {
@@ -50,7 +52,7 @@
     },
     'repository': {
         'source': 'geonode.catalogue.backends.pycsw_plugin.GeoNodeRepository',
-        'filter': 'is_published = %s' % true_value,
+        'filter': 'is_published = %s and dirty_state = %s ' % (true_value, false_value),
         'mappings': os.path.join(os.path.dirname(__file__), 'pycsw_local_mappings.py')
     }
 }

geonode/celery_app.py

@@ -21,18 +21,39 @@
 from __future__ import absolute_import
 
 import os
+import logging
+
 from celery import Celery
+# from celery.schedules import crontab
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'geonode.settings')
-
 app = Celery('geonode')
+logger = logging.getLogger(__name__)
+
+
+def _log(msg, *args):
+    logger.info(msg, *args)
+
 
 # Using a string here means the worker will not have to
 # pickle the object when using Windows.
 app.config_from_object('django.conf:settings', namespace="CELERY")
 app.autodiscover_tasks()
 
+""" CELERAY SAMPLE TASKS
+@app.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    # Calls test('hello') every 10 seconds.
+    sender.add_periodic_task(10.0, test.s('hello'), name='add every 10')
+
+    # Calls test('world') every 30 seconds
+    sender.add_periodic_task(30.0, test.s('world'), expires=10)
+
+@app.task
+def test(arg):
+    _log(arg)
 
 @app.task(bind=True)
 def debug_task(self):
-    print("Request: {!r}".format(self.request))
+    _log("Request: {!r}".format(self.request))
+"""

geonode/geoserver/helpers.py

@@ -66,6 +66,7 @@
 from geonode.layers.models import Layer, Attribute, Style
 from geonode.security.views import _perms_info_json
 from geonode.utils import set_attributes
+from geonode.security.utils import set_geowebcache_invalidate_cache
 import xml.etree.ElementTree as ET
 from django.utils.module_loading import import_string
 
@@ -345,6 +346,7 @@ def set_layer_style(saved_layer, title, sld, base_file=None):
                 layer.default_style = style
                 cat.save(layer)
                 saved_layer.default_style = save_style(style)
+                set_geowebcache_invalidate_cache(saved_layer.alternate)
         except Exception as e:
             logger.exception(e)
     else:
@@ -360,6 +362,7 @@ def set_layer_style(saved_layer, title, sld, base_file=None):
                 layer.default_style = style
                 cat.save(layer)
                 saved_layer.default_style = save_style(style)
+                set_geowebcache_invalidate_cache(saved_layer.alternate)
         except Exception as e:
             logger.exception(e)
 

geonode/geoserver/urls.py

@@ -49,8 +49,8 @@
                                                        downstream_path='wfs'), name='wfs_endpoint'),
     url(r'^wcs', views.geoserver_protected_proxy, dict(proxy_path='/gs/wcs',
                                                        downstream_path='wcs'), name='wcs_endpoint'),
-    url(r'^wps', views.geoserver_protected_proxy, dict(proxy_path='/gs/wps',
-                                                       downstream_path='wps'), name='wps_endpoint'),
+    url(r'^wps', views.geoserver_proxy, dict(proxy_path='/gs/wps',
+                                             downstream_path='wps'), name='wps_endpoint'),
     url(r'^pdf', views.geoserver_proxy, dict(proxy_path='/gs/pdf',
                                              downstream_path='pdf'), name='pdf_endpoint'),
     url(r'^(?P<workspace>[^/]*)/(?P<layername>[^/]*)/wms',

geonode/layers/templates/layers/layer_detail.html

@@ -655,6 +655,16 @@ <h4>{% trans "Styles" %}</h4>
     </li>
     {% endif %}
 
+    {% if OGC_SERVER.default.BACKEND == 'geonode.geoserver' %}
+        {% if "change_resourcebase" in perms_list %}
+        <li class="list-group-item">
+          <h4>{% trans "Clear the Server Cache of this layer" %}</h4>
+          <p>{% trans "Click the button below to wipe the tile-cache of this layer." %}</p>
+            <a href="#" class="btn btn-primary btn-block" data-dismiss="modal" id="tiledlayer_refresh">{% trans "Empty Tiled-Layer Cache" %}</a>
+        </li>
+        {% endif %}
+    {% endif %}
+
     {% if GEONODE_SECURITY_ENABLED %}
     {% if "change_resourcebase_permissions" in perms_list %}
     <li class="list-group-item">

geonode/layers/tests.py

@@ -123,6 +123,7 @@ def test_upload_layer(self):
         # Test redirection to login form when not logged in
         response = self.client.get(reverse('layer_upload'))
         self.assertEquals(response.status_code, 302)
+
         # Test return of upload form when logged in
         self.client.login(username="bobby", password="bob")
         response = self.client.get(reverse('layer_upload'))

geonode/layers/views.py

@@ -211,7 +211,7 @@ def layer_upload(request, template='upload/layer_upload.html'):
                 else:
                     saved_layer = Layer.objects.get(alternate=title)
                     if not saved_layer:
-                        msg = 'Failed to process.  Could not find matching layer.'
+                        msg = 'Failed to process. Could not find matching layer.'
                         raise Exception(msg)
                     sld = open(base_file).read()
                     set_layer_style(saved_layer, title, base_file, sld)