36 changes: 26 additions & 10 deletions geonode/geoserver/management/commands/sync_geonode_layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,10 @@
from geonode.geoserver.helpers import set_attributes_from_geoserver


def sync_geonode_layers(ignore_errors, filter, username, updatepermissions, updatethumbnails):
def sync_geonode_layers(ignore_errors, filter, username,
updatepermissions,
updatethumbnails,
updateattributes):
layers = Layer.objects.all().order_by('name')
if filter:
layers = layers.filter(name__icontains=filter)
Expand All @@ -42,15 +45,16 @@ def sync_geonode_layers(ignore_errors, filter, username, updatepermissions, upda
try:
count += 1
print 'Syncing layer %s/%s: %s' % (count, layers_count, layer.name)
if ast.literal_eval(updatepermissions):
if updatepermissions:
print 'Syncing permissions...'
# sync permissions in GeoFence
perm_spec = json.loads(_perms_info_json(layer))
# re-sync GeoFence security rules
layer.set_permissions(perm_spec)
if updateattributes:
# recalculate the layer statistics
set_attributes_from_geoserver(layer, overwrite=True)
if ast.literal_eval(updatethumbnails):
if updatethumbnails:
print 'Regenerating thumbnails...'
layer.save()
except Exception:
Expand All @@ -60,6 +64,8 @@ def sync_geonode_layers(ignore_errors, filter, username, updatepermissions, upda
if ignore_errors:
pass
else:
import traceback
traceback.print_exc()
print 'Stopping process because --ignore-errors was not set and an error was found.'
return
print 'There are %s layers which could not be updated because of errors' % len(layer_errors)
Expand All @@ -84,31 +90,41 @@ def add_arguments(self, parser):
'--filter',
dest="filter",
default=None,
help="Only update data the layers that match the given filter"),
help="Only update data the layers that match the given filter."),
parser.add_argument(
'-u',
'--username',
dest="username",
default=None,
help="Only update data owned by the specified username")
help="Only update data owned by the specified username.")
parser.add_argument(
'--updatepermissions',
action='store_true',
dest="updatepermissions",
default='True',
help="Update only the layer permissions. Does not regenerate styles and thumbnails")
default=False,
help="Update the layer permissions.")
parser.add_argument(
'--updatethumbnails',
action='store_true',
dest="updatethumbnails",
default='True',
help="Update only the layer styles and thumbnails. Does not re-sync security rules.")
default=False,
help="Update the layer styles and thumbnails.")
parser.add_argument(
'--updateattributes',
action='store_true',
dest="updateattributes",
default=False,
help="Update the layer attributes.")

def handle(self, **options):
ignore_errors = options.get('ignore_errors')
updatepermissions = options.get('updatepermissions')
updatethumbnails = options.get('updatethumbnails')
updateattributes = options.get('updateattributes')
filter = options.get('filter')
if not options.get('username'):
username = None
else:
username = options.get('username')
sync_geonode_layers(ignore_errors, filter, username, updatepermissions, updatethumbnails)
sync_geonode_layers(ignore_errors, filter, username,
updatepermissions, updatethumbnails, updateattributes)
34 changes: 25 additions & 9 deletions geonode/geoserver/signals.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ def geoserver_post_save_local(instance, *args, **kwargs):
gs_resource = None
values = None
_tries = 0
_max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 5)

# If the store in None then it's a new instance from an upload,
# only in this case run the geoserver_upload method
Expand All @@ -143,13 +144,23 @@ def geoserver_post_save_local(instance, *args, **kwargs):
charset=instance.charset)

def fetch_gs_resource(values, tries):
_max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 5)
gs_resource = gs_catalog.get_resource(
name=instance.name,
store=instance.store,
workspace=instance.workspace)
if not gs_resource:
gs_resource = gs_catalog.get_resource(name=instance.alternate)
try:
gs_resource = gs_catalog.get_resource(
name=instance.name,
store=instance.store,
workspace=instance.workspace)
except BaseException:
try:
gs_resource = gs_catalog.get_resource(
name=instance.alternate,
store=instance.store,
workspace=instance.workspace)
except BaseException:
try:
gs_resource = gs_catalog.get_resource(
name=instance.alternate or instance.typename)
except BaseException:
gs_resource = None

if gs_resource:
gs_resource.title = instance.title or ""
Expand All @@ -167,13 +178,14 @@ def fetch_gs_resource(values, tries):
msg = "There isn't a geoserver resource for this layer: %s" % instance.name
logger.exception(msg)
if tries >= _max_tries:
raise GeoNodeException(msg)
# raise GeoNodeException(msg)
return (values, None)
gs_resource = None
sleep(3.00)

return (values, gs_resource)

while not gs_resource:
while not gs_resource and _tries < _max_tries:
values, gs_resource = fetch_gs_resource(values, _tries)
_tries += 1

Expand All @@ -183,6 +195,7 @@ def fetch_gs_resource(values, tries):
metadata_links.append((link.mime, link.name, link.url))

if gs_resource:
logger.info("Found geoserver resource for this layer: %s" % instance.name)
gs_resource.metadata_links = metadata_links
# gs_resource should only be called if
# ogc_server_settings.BACKEND_WRITE_ENABLED == True
Expand Down Expand Up @@ -217,6 +230,9 @@ def fetch_gs_resource(values, tries):
'try to use: "%s"' % (gs_resource, str(e)))
e.args = (msg,)
logger.exception(e)
else:
msg = "There isn't a geoserver resource for this layer: %s" % instance.name
logger.warn(msg)

if isinstance(instance, ResourceBase):
if hasattr(instance, 'layer'):
Expand Down
18 changes: 11 additions & 7 deletions geonode/layers/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -505,13 +505,17 @@ def _get_alternate_name(instance):

def pre_save_layer(instance, sender, **kwargs):
if kwargs.get('raw', False):
instance.owner = instance.resourcebase_ptr.owner
instance.uuid = instance.resourcebase_ptr.uuid
instance.bbox_x0 = instance.resourcebase_ptr.bbox_x0
instance.bbox_x1 = instance.resourcebase_ptr.bbox_x1
instance.bbox_y0 = instance.resourcebase_ptr.bbox_y0
instance.bbox_y1 = instance.resourcebase_ptr.bbox_y1
instance.srid = instance.resourcebase_ptr.srid
try:
_resourcebase_ptr = instance.resourcebase_ptr
instance.owner = _resourcebase_ptr.owner
instance.uuid = _resourcebase_ptr.uuid
instance.bbox_x0 = _resourcebase_ptr.bbox_x0
instance.bbox_x1 = _resourcebase_ptr.bbox_x1
instance.bbox_y0 = _resourcebase_ptr.bbox_y0
instance.bbox_y1 = _resourcebase_ptr.bbox_y1
instance.srid = _resourcebase_ptr.srid
except BaseException as e:
logger.exception(e)

if instance.abstract == '' or instance.abstract is None:
instance.abstract = unicode(_('No abstract provided'))
Expand Down
6 changes: 4 additions & 2 deletions geonode/security/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ def set_permissions(self, perm_spec):
assign_perm(perm, user, self.layer)
else:
assign_perm(perm, user, self.get_self_resource())
# Set the GeoFence Owner Rules
# Set the GeoFence Rules
geofence_user = str(user)
if "AnonymousUser" in geofence_user:
geofence_user = None
Expand All @@ -223,7 +223,9 @@ def set_permissions(self, perm_spec):
assign_perm(perm, group, self.layer)
else:
assign_perm(perm, group, self.get_self_resource())
# Set the GeoFence Owner Rules
# Set the GeoFence Rules
if group.name == 'anonymous':
group = None
if settings.OGC_SERVER['default'].get("GEOFENCE_SECURITY_ENABLED", False):
if self.polymorphic_ctype.name == 'layer':
if getattr(settings, 'DELAYED_SECURITY_SIGNALS', False):
Expand Down
19 changes: 19 additions & 0 deletions geonode/services/migrations/0028_remove_service_last_updated.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2019-10-04 15:48
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

dependencies = [
('services', '0027_auto_20190429_0831'),
]

operations = [
migrations.RemoveField(
model_name='service',
name='last_updated',
),
]
19 changes: 19 additions & 0 deletions geonode/services/migrations/0029_remove_service_created.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.25 on 2019-10-04 15:59
from __future__ import unicode_literals

from django.db import migrations


class Migration(migrations.Migration):

dependencies = [
('services', '0028_remove_service_last_updated'),
]

operations = [
migrations.RemoveField(
model_name='service',
name='created',
),
]
6 changes: 0 additions & 6 deletions geonode/services/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,12 +127,6 @@ class Service(ResourceBase):
settings.AUTH_USER_MODEL,
through='ServiceProfileRole'
)
created = models.DateTimeField(
auto_now_add=True
)
last_updated = models.DateTimeField(
auto_now=True
)
first_noanswer = models.DateTimeField(
null=True,
blank=True
Expand Down
4 changes: 2 additions & 2 deletions geonode/tests/csw.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,8 @@ def test_csw_outputschema_dc_bbox(self):
from decimal import Decimal
logger.debug([Decimal(record.bbox.minx), Decimal(record.bbox.miny),
Decimal(record.bbox.maxx), Decimal(record.bbox.maxy)])
self.assertEquals(Decimal(record.bbox.minx), Decimal('-81.8593555'))
self.assertEquals(Decimal(record.bbox.miny), Decimal('12.1665322'))
self.assertEquals(Decimal(record.bbox.minx), Decimal('-81.859356'))
self.assertEquals(Decimal(record.bbox.miny), Decimal('12.166532'))
self.assertEquals(Decimal(record.bbox.maxx), Decimal('-81.356409'))
self.assertEquals(Decimal(record.bbox.maxy), Decimal('13.396306'))

Expand Down
14 changes: 7 additions & 7 deletions geonode/upload/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ def __repr__(self):

types = [
FileType("Shapefile", "shp", vector,
auxillary_file_exts=('dbf', 'shx', 'prj')),
auxillary_file_exts=('dbf', 'shx', 'prj',)),
FileType("GeoTIFF", _tif_extensions[0], raster,
aliases=_tif_extensions[1:]),
FileType(
Expand All @@ -134,19 +134,19 @@ def __repr__(self):
auxillary_file_exts=_mosaics_extensions + _tif_extensions
),
FileType("ASCII Text File", "asc", raster,
auxillary_file_exts=('prj')),
auxillary_file_exts=('prj',)),
# requires geoserver importer extension
FileType("PNG", "png", raster,
auxillary_file_exts=('prj')),
auxillary_file_exts=('prj',)),
FileType("JPG", "jpg", raster,
auxillary_file_exts=('prj')),
auxillary_file_exts=('prj',)),
FileType("CSV", "csv", vector),
FileType("GeoJSON", "geojson", vector),
FileType("KML", "kml", vector),
FileType(
"KML Ground Overlay", "kml-overlay", raster,
aliases=("kmz", "kml"),
auxillary_file_exts=("png", "gif", "jpg") + _tif_extensions
aliases=("kmz", "kml",),
auxillary_file_exts=("png", "gif", "jpg",) + _tif_extensions
),
# requires geoserver gdal extension
FileType("ERDASImg", "img", raster),
Expand Down Expand Up @@ -174,7 +174,7 @@ def __repr__(self):
aliases=('tl2', 'tl3', 'tl4', 'tl5', 'tl6', 'tl7', 'tl8', 'tl9')),
# requires gdal plugin for mrsid and jp2
FileType("MrSID", "sid", raster,
auxillary_file_exts=('sdw')),
auxillary_file_exts=('sdw',)),
FileType("JP2", "jp2", raster)
]

Expand Down
Binary file added geonode/upload/tests/data/arc_sample.zip
Binary file not shown.
366 changes: 366 additions & 0 deletions geonode/upload/tests/data/arc_sample/precip30min.asc

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions geonode/upload/tests/data/arc_sample/precip30min.prj
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
GEOGCS["WGS 84",
DATUM["World Geodetic System 1984",
SPHEROID["WGS 84", 6378137.0, 298.257223563, AUTHORITY["EPSG","7030"]],
AUTHORITY["EPSG","6326"]],
PRIMEM["Greenwich", 0.0, AUTHORITY["EPSG","8901"]],
UNIT["degree", 0.017453292519943295],
AXIS["Geodetic longitude", EAST],
AXIS["Geodetic latitude", NORTH],
AUTHORITY["EPSG","4326"]]
50 changes: 40 additions & 10 deletions geonode/upload/tests/integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,16 +348,28 @@ def check_invalid_projection(self, layer_name, resp, data):
self.assertTrue(resp.code, 200)
if not isinstance(data, basestring):
self.assertTrue(data['success'])
self.assertTrue(upload_step("srs") in data['redirect_to'])
resp, soup = self.client.get_html(data['redirect_to'])
# grab an h2 and find the name there as part of a message saying it's
# bad
h2 = soup.find_all(['h2'])[0]
self.assertTrue(str(h2).find(layer_name))
srs_step = upload_step("srs")
if "srs" in data['redirect_to']:
self.assertTrue(srs_step in data['redirect_to'])
resp, soup = self.client.get_html(data['redirect_to'])
# grab an h2 and find the name there as part of a message saying it's
# bad
h2 = soup.find_all(['h2'])[0]
self.assertTrue(str(h2).find(layer_name))

def check_upload_complete(self, layer_name, resp, data):
""" Makes sure that we got the correct response from an layer
that can't be uploaded"""
self.assertTrue(resp.code, 200)
if not isinstance(data, basestring):
self.assertTrue(data['success'])
final_step = upload_step("final")
if "final" in data['redirect_to']:
self.assertTrue(final_step in data['redirect_to'])

def upload_folder_of_files(self, folder, final_check, session_ids=None):

mains = ('.tif', '.shp', '.zip')
mains = ('.tif', '.shp', '.zip', '.asc')

def is_main(_file):
_, ext = os.path.splitext(_file)
Expand Down Expand Up @@ -525,17 +537,34 @@ def test_zipped_upload(self):
for f in glob.glob(fpath):
zf.write(f, os.path.basename(f))
zf.close()
self.upload_file(abspath, self.complete_upload,
self.upload_file(abspath,
self.complete_upload,
check_name='san_andres_y_providencia_poi')

def test_ascii_grid_upload(self):
""" Tests the layers that ASCII grid files are uploaded along with aux"""
session_ids = []

PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
thelayer_path = os.path.join(
PROJECT_ROOT,
'data/arc_sample')
self.upload_folder_of_files(
thelayer_path,
self.complete_raster_upload,
session_ids=session_ids)

def test_invalid_layer_upload(self):
""" Tests the layers that are invalid and should not be uploaded"""
# this issue with this test is that the importer supports
# shapefiles without an .prj
session_ids = []

invalid_path = os.path.join(BAD_DATA)
self.upload_folder_of_files(
invalid_path,
self.check_invalid_projection)
self.check_invalid_projection,
session_ids=session_ids)

def test_coherent_importer_session(self):
""" Tests that the upload computes correctly next session IDs"""
Expand All @@ -553,7 +582,8 @@ def test_coherent_importer_session(self):
invalid_path = os.path.join(BAD_DATA)
self.upload_folder_of_files(
invalid_path,
self.check_invalid_projection, session_ids=session_ids)
self.check_invalid_projection,
session_ids=session_ids)

# Finally try to upload a good file anc check the session IDs
fname = os.path.join(GOOD_DATA, 'raster', 'relief_san_andres.tif')
Expand Down
17 changes: 11 additions & 6 deletions geonode/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,13 @@ def bbox_to_wkt(x0, x1, y0, y1, srid="4326"):
if srid and str(srid).startswith('EPSG:'):
srid = srid[5:]
if None not in [x0, x1, y0, y1]:
wkt = 'SRID=%s;POLYGON((%s %s,%s %s,%s %s,%s %s,%s %s))' % (
srid, x0, y0, x0, y1, x1, y1, x1, y0, x0, y0)
wkt = 'SRID=%s;POLYGON((%f %f,%f %f,%f %f,%f %f,%f %f))' % (
srid,
float(x0), float(y0),
float(x0), float(y1),
float(x1), float(y1),
float(x1), float(y0),
float(x0), float(y0))
else:
wkt = 'SRID=4326;POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))'
return wkt
Expand Down Expand Up @@ -1367,7 +1372,7 @@ def __init__(self):
self.username = ogc_server_settings['USER'] if 'USER' in ogc_server_settings else 'admin'
self.password = ogc_server_settings['PASSWORD'] if 'PASSWORD' in ogc_server_settings else 'geoserver'

def request(self, url, method='GET', data=None, headers={}, stream=False, timeout=None, user=None):
def request(self, url, method='GET', data=None, headers={}, stream=False, timeout=None, retries=None, user=None):
if (user or self.username != 'admin') and \
check_ogc_backend(geoserver.BACKEND_PACKAGE) and 'Authorization' not in headers:
if connection.cursor().db.vendor not in ('sqlite', 'sqlite3', 'spatialite'):
Expand All @@ -1391,9 +1396,9 @@ def request(self, url, method='GET', data=None, headers={}, stream=False, timeou
content = None
session = requests.Session()
retry = Retry(
total=self.retries,
read=self.retries,
connect=self.retries,
total=retries or self.retries,
read=retries or self.retries,
connect=retries or self.retries,
backoff_factor=self.backoff_factor,
status_forcelist=self.status_forcelist,
)
Expand Down