Skip to content

Commit

Permalink
Fix for ISSUE GeoNode#2975 - Upload Layer Issue GeoNode#2975
Browse files Browse the repository at this point in the history 
 - Fix kombu dep

 - Remove kombu dependencies

 - merge

 - Fix PR for ISSUE GeoNode#3035
  • Loading branch information
@travislbrundage
afabiani authored and travislbrundage committed Oct 21, 2017
1 parent f4f0cdb commit 81ab7f5
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 46 deletions.
61 changes: 43 additions & 18 deletions geonode/geoserver/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,16 +183,34 @@ def _style_name(resource):
return _punc.sub("_", resource.store.workspace.name + ":" + resource.name)


def get_sld_for(layer):
# FIXME: GeoServer sometimes fails to associate a style with the data, so
def get_sld_for(gs_catalog, layer):
# GeoServer sometimes fails to associate a style with the data, so
# for now we default to using a point style.(it works for lines and
# polygons, hope this doesn't happen for rasters though)
name = layer.default_style.name if layer.default_style is not None else "point"
if layer.default_style is None:
gs_catalog._cache.clear()
layer = gs_catalog.get_layer(layer.name)
name = layer.default_style.name if layer.default_style is not None else "raster"

# Detect geometry type if it is a FeatureType
if layer.resource.resource_type == 'featureType':
res = layer.resource
res.fetch()
ft = res.store.get_resources(res.name)
ft.fetch()
for attr in ft.dom.find("attributes").getchildren():
attr_binding = attr.find("binding")
if "jts.geom" in attr_binding.text:
if "Polygon" in attr_binding.text:
name = "polygon"
elif "Line" in attr_binding.text:
name = "line"
else:
name = "point"

# FIXME: When gsconfig.py exposes the default geometry type for vector
# layers we should use that rather than guessing based on the auto-detected
# style.

if name in _style_templates:
fg, bg, mark = _style_contexts.next()
return _style_templates[name] % dict(
Expand All @@ -213,7 +231,7 @@ def fixup_style(cat, resource, style):
logger.info("%s uses a default style, generating a new one", lyr)
name = _style_name(resource)
if style is None:
sld = get_sld_for(lyr)
sld = get_sld_for(cat, lyr)
else:
sld = style.read()
logger.info("Creating style [%s]", name)
Expand Down Expand Up @@ -378,7 +396,7 @@ def gs_slurp(
if console is None:
console = open(os.devnull, 'w')

if verbosity > 1:
if verbosity > 0:
print >> console, "Inspecting the available layers in GeoServer ..."
cat = Catalog(ogc_server_settings.internal_rest, _user, _password)
layergroups = []
Expand Down Expand Up @@ -442,7 +460,7 @@ def gs_slurp(
# disabled_resources = [k for k in resources if k.enabled == "false"]

number = len(resources)
if verbosity > 1:
if verbosity > 0:
msg = "Found %d layers, starting processing" % number
print >> console, msg
output = {
Expand Down Expand Up @@ -654,7 +672,7 @@ def gs_slurp(
deleted_layers.append(layer)

number_deleted = len(deleted_layers)
if verbosity > 1:
if verbosity > 0:
msg = "\nFound %d layers to delete, starting processing" % number_deleted if number_deleted > 0 else \
"\nFound %d layers to delete" % number_deleted
print >> console, msg
Expand Down Expand Up @@ -1252,27 +1270,34 @@ def geoserver_upload(
sld = f.read()
f.close()
else:
sld = get_sld_for(publishing)
sld = get_sld_for(cat, publishing)

style = None
if sld is not None:
try:
cat.create_style(name, sld)
style = cat.get_style(name)
except geoserver.catalog.ConflictingDataError as e:
msg = ('There was already a style named %s in GeoServer, '
'try to use: "%s"' % (name + "_layer", str(e)))
logger.warn(msg)
e.args = (msg,)
try:
cat.create_style(name + '_layer', sld)

style = cat.get_style(name)
if style is None:
try:
cat.create_style(name + '_layer', sld)
except geoserver.catalog.ConflictingDataError as e:
msg = ('There was already a style named %s in GeoServer, '
'cannot overwrite: "%s"' % (name, str(e)))
logger.warn(msg)
e.args = (msg,)

style = cat.get_style(name + "_layer")
except geoserver.catalog.ConflictingDataError as e:
style = cat.get_style('point')
msg = ('There was already a style named %s in GeoServer, '
'cannot overwrite: "%s"' % (name, str(e)))
logger.error(msg)
e.args = (msg,)
if style is None:
style = cat.get_style('point')
msg = ('Could not find any suitable style in GeoServer '
'for Layer: "%s"' % (name))
logger.error(msg)

# FIXME: Should we use the fully qualified typename?
publishing.default_style = style
Expand Down
59 changes: 53 additions & 6 deletions geonode/security/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#########################################################################
try:
import json
except ImportError:
from django.utils import simplejson as json
import logging
import traceback
import requests
Expand Down Expand Up @@ -253,11 +257,54 @@ def set_geofence_all(instance):
resource = instance.get_self_resource()

if hasattr(resource, "layer"):
"""
curl -X POST -u admin:geoserver -H "Content-Type: text/xml" -d \
"<Rule><workspace>geonode</workspace><layer>{layer}</layer><access>ALLOW</access></Rule>" \
http://<host>:<port>/geoserver/geofence/rest/rules
"""
try:
url = settings.OGC_SERVER['default']['LOCATION']
user = settings.OGC_SERVER['default']['USER']
passwd = settings.OGC_SERVER['default']['PASSWORD']
# Check first that the rules does not exist already
"""
curl -X GET -u admin:geoserver -H "Content-Type: application/json" \
http://<host>:<port>/geoserver/geofence/rest/rules.json?layer=<layer_name>
"""
headers = {'Content-type': 'application/json'}
r = requests.get(url + 'geofence/rest/rules.json?layer=' + resource.layer.name,
headers=headers,
auth=HTTPBasicAuth(user, passwd))

rules_already_present = False
if (r.status_code != 200):
logger.warning("Could not GET GeoServer Rules for Layer " + str(resource.layer.name))
else:
try:
rules_objs = json.loads(r.text)
rules_count = rules_objs['count']
rules = rules_objs['rules']
if rules_count > 1:
for rule in rules:
if rule['userName'] is None and rule['access'] == 'ALLOW':
rules_already_present = True
except:
tb = traceback.format_exc()
logger.debug(tb)

# Create GeoFence Rules for ANONYMOUS to the Layer
"""
curl -X POST -u admin:geoserver -H "Content-Type: text/xml" -d \
"<Rule><workspace>geonode</workspace><layer>{layer}</layer><access>ALLOW</access></Rule>" \
http://<host>:<port>/geoserver/geofence/rest/rules
"""
headers = {'Content-type': 'application/xml'}
payload = "<Rule><workspace>geonode</workspace><layer>"
payload = payload + resource.layer.name
payload = payload + "</layer><access>ALLOW</access></Rule>"

if not rules_already_present:
r = requests.post(url + 'geofence/rest/rules',
headers=headers,
data=payload,
auth=HTTPBasicAuth(user, passwd))
if (r.status_code != 200):
logger.warning("Could not ADD GeoServer ANONYMOUS Rule for Layer " + str(resource.layer.name))

payload = "<Rule><workspace>{}</workspace><layer>".format(resource.layer.workspace)
payload = payload + resource.layer.name
Expand Down Expand Up @@ -317,7 +364,7 @@ def set_owner_permissions(resource):
"""assign all admin permissions to the owner"""
if resource.polymorphic_ctype:
if resource.polymorphic_ctype.name == 'layer':
# TODO: Assign GeoFence Layer Access to Owner
# Assign GeoFence Layer Access to Owner
for perm in LAYER_ADMIN_PERMISSIONS:
assign_perm(perm, resource.owner, resource.layer)
Expand Down
12 changes: 2 additions & 10 deletions geonode/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
# Django settings for the GeoNode project.
import os

from kombu import Queue
from geonode import __file__ as geonode_path
from geonode import get_version
from geonode.celery_app import app # flake8: noqa
Expand Down Expand Up @@ -316,7 +315,9 @@
'autocomplete_light',
'mptt',
# 'modeltranslation',
# 'djkombu',
'djcelery',
# 'kombu.transport.django',
'storages',

# Theme
Expand Down Expand Up @@ -972,15 +973,6 @@
'geonode.tasks.email'
)


CELERY_QUEUES = [
Queue('default', routing_key='default'),
Queue('cleanup', routing_key='cleanup'),
Queue('update', routing_key='update'),
Queue('email', routing_key='email'),
]


# AWS S3 Settings

S3_STATIC_ENABLED = os.environ.get('S3_STATIC_ENABLED', False)
Expand Down
4 changes: 4 additions & 0 deletions geonode/tasks/deletion.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,17 @@
from geonode.layers.models import Layer
from geonode.maps.models import Map
from celery.task import task
from celery.utils.log import get_task_logger

logger = get_task_logger(__name__)


@task(name='geonode.tasks.deletion.delete_layer', queue='cleanup')
def delete_layer(object_id):
"""
Deletes a layer.
"""
logger.info('Deleting Layer ID {0}'.format(object_id))
try:
layer = Layer.objects.get(id=object_id)
except Layer.DoesNotExist:
Expand Down
30 changes: 18 additions & 12 deletions geonode/upload/upload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,36 +606,42 @@ def final_step(upload_session, user):
sld = f.read()
f.close()
else:
sld = get_sld_for(publishing)
sld = get_sld_for(cat, publishing)

style = None
print " **************************************** "
if sld is not None:
try:
cat.create_style(name, sld)
style = cat.get_style(name)
except geoserver.catalog.ConflictingDataError as e:
msg = 'There was already a style named %s in GeoServer, try using another name: "%s"' % (
name, str(e))
try:
cat.create_style(name + '_layer', sld)
style = cat.get_style(name + '_layer')
except geoserver.catalog.ConflictingDataError as e:
msg = 'There was already a style named %s in GeoServer, cannot overwrite: "%s"' % (
name, str(e))
logger.error(msg)
e.args = (msg,)

if style is None:
try:
style = cat.get_style(name)
except:
logger.warn('Could not retreive the Layer default Style name')
# what are we doing with this var?
msg = 'No style could be created for the layer, falling back to POINT default one'
style = cat.get_style('point')
logger.warn(msg)
e.args = (msg,)

# FIXME: Should we use the fully qualified typename?
publishing.default_style = style
_log('default style set to %s', name)
cat.save(publishing)
try:
style = cat.get_style(name + '_layer')
except:
style = cat.get_style('point')
logger.warn(msg)
e.args = (msg,)

if style:
# FIXME: Should we use the fully qualified typename?
publishing.default_style = style
_log('default style set to %s', name)
cat.save(publishing)

_log('Creating Django record for [%s]', name)
target = task.target
Expand Down

0 comments on commit 81ab7f5

Please sign in to comment.