Skip to content

Commit

Permalink
use requests retry to improve geonode/geoserver connection
Browse files Browse the repository at this point in the history
  • Loading branch information
hisham waleed karam committed Jan 30, 2019
1 parent 77e0fe3 commit 7d07b84
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 79 deletions.
98 changes: 31 additions & 67 deletions geonode/geoserver/helpers.py
View file
Expand Up @@ -33,12 +33,11 @@
from threading import local
import time
import uuid
import base64
import httplib2

import urllib
from urlparse import urlsplit, urlparse, urljoin

from.utils import geoserver_requests_session
from agon_ratings.models import OverallRating
from bs4 import BeautifulSoup
from dialogos.models import Comment
Expand Down Expand Up @@ -86,11 +85,11 @@ def check_geoserver_is_up():
this is needed to be able to upload.
"""
url = "%s" % ogc_server_settings.LOCATION
resp, content = http_client.request(url, "GET")
req = http_client.get(url)
msg = ('Cannot connect to the GeoServer at %s\nPlease make sure you '
'have started it.' % url)
logger.debug(resp)
assert resp['status'] == '200', msg
logger.debug(req)
assert req.status_code == 200, msg


def _add_sld_boilerplate(symbolizer):
Expand Down Expand Up @@ -409,9 +408,9 @@ def cascading_delete(cat, layer_name):
raise e

if resource is None:
# If there is no associated resource,
# this method can not delete anything.
# Let's return and make a note in the log.
# If there is no associated resource,
# this method can not delete anything.
# Let's return and make a note in the log.
logger.debug(
'cascading_delete was called with a non existent resource')
return
Expand Down Expand Up @@ -818,7 +817,8 @@ def set_attributes_from_geoserver(layer, overwrite=False):
dft_url = server_url + ("%s?f=json" % layer.alternate)
try:
# The code below will fail if http_client cannot be imported
body = json.loads(http_client.request(dft_url)[1])
req = http_client.get(dft_url)
body = req.json()
attribute_map = [[n["name"], _esri_types[n["type"]]]
for n in body["fields"] if n.get("name") and n.get("type")]
except Exception:
Expand All @@ -834,7 +834,8 @@ def set_attributes_from_geoserver(layer, overwrite=False):
try:
# The code below will fail if http_client cannot be imported or
# WFS not supported
body = http_client.request(dft_url)[1]
req = http_client.get(dft_url)
body = req.content
doc = etree.fromstring(body)
path = ".//{xsd}extension/{xsd}sequence/{xsd}element".format(
xsd="{http://www.w3.org/2001/XMLSchema}")
Expand All @@ -859,7 +860,8 @@ def set_attributes_from_geoserver(layer, overwrite=False):
"y": 1
})
try:
body = http_client.request(dft_url)[1]
req = http_client.get(dft_url)
body = req.content
soup = BeautifulSoup(body)
for field in soup.findAll('th'):
if(field.string is None):
Expand All @@ -878,7 +880,8 @@ def set_attributes_from_geoserver(layer, overwrite=False):
"identifiers": layer.alternate.encode('utf-8')
})
try:
response, body = http_client.request(dc_url)
req = http_client.get(dc_url)
body = req.content
doc = etree.fromstring(body)
path = ".//{wcs}Axis/{wcs}AvailableKeys/{wcs}Key".format(
wcs="{http://www.opengis.net/wcs/1.1.1}")
Expand Down Expand Up @@ -1406,21 +1409,8 @@ def all(self):
def get_wms():
wms_url = ogc_server_settings.internal_ows + \
"?service=WMS&request=GetCapabilities&version=1.1.0"
netloc = urlparse(wms_url).netloc
http = httplib2.Http()
http.add_credentials(_user, _password)
http.authorizations.append(
httplib2.BasicAuthentication(
(_user, _password),
netloc,
wms_url,
{},
None,
None,
http
)
)
body = http.request(wms_url)[1]
req = http_client.get(wms_url)
body = req.content
_wms = WebMapService(wms_url, xml=body)
return _wms

Expand Down Expand Up @@ -1486,22 +1476,16 @@ def wps_execute_layer_attribute_statistics(layer_name, field):


def _stylefilterparams_geowebcache_layer(layer_name):
http = httplib2.Http()
username, password = ogc_server_settings.credentials
auth = base64.encodestring(username + ':' + password)
# http.add_credentials(username, password)
headers = {
"Content-Type": "text/xml",
"Authorization": "Basic " + auth
"Content-Type": "text/xml"
}
url = '%sgwc/rest/layers/%s.xml' % (ogc_server_settings.LOCATION, layer_name)

# read GWC configuration
method = "GET"
response, _ = http.request(url, method, headers=headers)
if response.status != 200:
req = http_client.get(url,headers=headers)
if req.status_code != 200:
line = "Error {0} reading Style Filter Params GeoWebCache at {1}".format(
response.status, url
req.status_code, url
)
logger.error(line)
return
Expand All @@ -1519,35 +1503,29 @@ def _stylefilterparams_geowebcache_layer(layer_name):
param_filters[0].append(style_filters_elem)
body = ET.tostring(tree)
if body:
method = "POST"
response, _ = http.request(url, method, body=body, headers=headers)
if response.status != 200:
req = http_client.post(url,data=body, headers=headers)
if req.status_code != 200:
line = "Error {0} writing Style Filter Params GeoWebCache at {1}".format(
response.status, url
req.status_code, url
)
logger.error(line)


def _invalidate_geowebcache_layer(layer_name, url=None):
http = httplib2.Http()
username, password = ogc_server_settings.credentials
auth = base64.encodestring(username + ':' + password)
# http.add_credentials(username, password)
method = "POST"
headers = {
"Content-Type": "text/xml",
"Authorization": "Basic " + auth
}
body = """
<truncateLayer><layerName>{0}</layerName></truncateLayer>
""".strip().format(layer_name)
if not url:
url = '%sgwc/rest/masstruncate' % ogc_server_settings.LOCATION
response, _ = http.request(url, method, body=body, headers=headers)
req = http_client.post(url, data=body, headers=headers)

if response.status != 200:
if req.status_code != 200:
line = "Error {0} invalidating GeoWebCache at {1}".format(
response.status, url
req.status_code, url
)
logger.error(line)

Expand Down Expand Up @@ -1733,21 +1711,7 @@ def get_time_info(layer):
_csw = None
_user, _password = ogc_server_settings.credentials

http_client = httplib2.Http()
http_client.add_credentials(_user, _password)
http_client.add_credentials(_user, _password)
_netloc = urlparse(ogc_server_settings.LOCATION).netloc
http_client.authorizations.append(
httplib2.BasicAuthentication(
(_user, _password),
_netloc,
ogc_server_settings.LOCATION,
{},
None,
None,
http_client
)
)
http_client = geoserver_requests_session()


url = ogc_server_settings.rest
Expand Down Expand Up @@ -1807,9 +1771,9 @@ def _render_thumbnail(req_body, width=240, height=180):
data = data.encode('ASCII', 'ignore')
data = unicode(data, errors='ignore').encode('UTF-8')
try:
resp, content = http_client.request(url, "POST", data, {
'Content-type': 'text/html'
})
req = http_client.post(
url, data=data, headers={'Content-type': 'text/html'})
content = req.content
except Exception:
logging.warning('Error generating thumbnail')
return
Expand Down
53 changes: 53 additions & 0 deletions geonode/geoserver/utils.py
View file
@@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
#########################################################################
#
# Copyright (C) 2016 OSGeo
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#########################################################################
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
requests.packages.urllib3.disable_warnings()


def requests_retry(retries=3,
backoff_factor=0.5,
status_forcelist=(500, 502, 504),
session=None):
session = session or requests.Session()
# disable ssl verify
session.verify = False
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
method_whitelist=frozenset(['GET', 'POST', 'PUT', 'DELETE', 'HEAD']))
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session


def geoserver_requests_session():
from .helpers import ogc_server_settings
_user, _password = ogc_server_settings.credentials
url = ogc_server_settings.LOCATION
session = requests.Session()
session.auth = (_user, _password)
session = requests_retry(session=session)
return session
26 changes: 14 additions & 12 deletions geonode/geoserver/views.py
View file
Expand Up @@ -22,7 +22,6 @@
import re
import json
import logging
import httplib2
import traceback
from lxml import etree
from os.path import isfile
Expand All @@ -43,7 +42,7 @@
from django.utils.translation import ugettext as _

from guardian.shortcuts import get_objects_for_user

from .utils import requests_retry
from geonode.base.models import ResourceBase
from geonode.layers.forms import LayerStyleUploadForm
from geonode.layers.models import Layer, Style
Expand Down Expand Up @@ -623,9 +622,9 @@ def list_item(lyr):
}

url = "%srest/process/batchDownload/launch/" % ogc_server_settings.LOCATION
resp, content = http_client.request(
url, 'POST', body=json.dumps(fake_map))
return HttpResponse(content, status=resp.status)
req = http_client.post(url, data=json.dumps(fake_map))
content = req.content
return HttpResponse(content, status=req.status_code)

if request.method == 'GET':
# essentially, this just proxies back to geoserver
Expand All @@ -635,8 +634,9 @@ def list_item(lyr):

url = "%srest/process/batchDownload/status/%s" % (
ogc_server_settings.LOCATION, download_id)
resp, content = http_client.request(url, 'GET')
return HttpResponse(content, status=resp.status)
req = http_client.get(url)
content = req.content
return HttpResponse(content, status=req.status_code)


def resolve_user(request):
Expand Down Expand Up @@ -760,18 +760,20 @@ def get_layer_capabilities(layer, version='1.3.0', access_token=None, tolerant=F
wms_url = '%s?service=wms&version=%s&request=GetCapabilities'\
% (layer.remote_service.service_url, version)

http = httplib2.Http()
response, getcap = http.request(wms_url)
if tolerant and ('ServiceException' in getcap or response.status == 404):
session = requests_retry()
req = session.get(wms_url)
getcap = req.content
if tolerant and ('ServiceException' in getcap or req.status_code == 404):
# WARNING Please make sure to have enabled DJANGO CACHE as per
# https://docs.djangoproject.com/en/2.0/topics/cache/#filesystem-caching
wms_url = '%s%s/ows?service=wms&version=%s&request=GetCapabilities&layers=%s'\
% (ogc_server_settings.public_url, workspace, version, layer)
if access_token:
wms_url += ('&access_token=%s' % access_token)
response, getcap = http.request(wms_url)
req = session.get(wms_url)
getcap = req.content

if 'ServiceException' in getcap or response.status == 404:
if 'ServiceException' in getcap or req.status_code == 404:
return None
return getcap

Expand Down

0 comments on commit 7d07b84

Please sign in to comment.