Skip to content

Commit

Permalink
[Minor improvements] Allow requests get to use timeouts / expose Moni…
Browse files Browse the repository at this point in the history 
…toring variables
  • Loading branch information
afabiani committed May 10, 2019
1 parent f053c13 commit 1974350
Show file tree
Hide file tree
Showing 6 changed files with 47 additions and 30 deletions.
9 changes: 6 additions & 3 deletions geonode/base/management/commands/backup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ def create_geoserver_backup(self, settings, target_folder):

gs_bk_exec_id = gs_backup['backup']['execution']['id']
r = requests.get(url + 'rest/br/backup/' + str(gs_bk_exec_id) + '.json',
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code == 200):

try:
Expand All @@ -130,7 +131,8 @@ def create_geoserver_backup(self, settings, target_folder):

gs_bk_exec_id = gs_backup['backup']['execution']['id']
r = requests.get(url + 'rest/br/backup/' + str(gs_bk_exec_id) + '.json',
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code == 200):
gs_bk_exec_status = gs_backup['backup']['execution']['status']
gs_bk_exec_progress = gs_backup['backup']['execution']['progress']
Expand All @@ -139,7 +141,8 @@ def create_geoserver_backup(self, settings, target_folder):
if (gs_bk_exec_progress != gs_bk_exec_progress_updated):
gs_bk_exec_progress_updated = gs_bk_exec_progress
r = requests.get(url + 'rest/br/backup/' + str(gs_bk_exec_id) + '.json',
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code == 200):

try:
Expand Down
9 changes: 6 additions & 3 deletions geonode/base/management/commands/restore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ def restore_geoserver_backup(self, settings, target_folder):

gs_bk_exec_id = gs_backup['restore']['execution']['id']
r = requests.get(url + 'rest/br/restore/' + str(gs_bk_exec_id) + '.json',
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code == 200):

try:
Expand All @@ -143,7 +144,8 @@ def restore_geoserver_backup(self, settings, target_folder):

gs_bk_exec_id = gs_backup['restore']['execution']['id']
r = requests.get(url + 'rest/br/restore/' + str(gs_bk_exec_id) + '.json',
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code == 200):
gs_bk_exec_status = gs_backup['restore']['execution']['status']
gs_bk_exec_progress = gs_backup['restore']['execution']['progress']
Expand All @@ -152,7 +154,8 @@ def restore_geoserver_backup(self, settings, target_folder):
if (gs_bk_exec_progress != gs_bk_exec_progress_updated):
gs_bk_exec_progress_updated = gs_bk_exec_progress
r = requests.get(url + 'rest/br/restore/' + str(gs_bk_exec_id) + '.json',
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code == 200):

try:
Expand Down
4 changes: 2 additions & 2 deletions geonode/contrib/monitoring/service_handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ def _collect(self, *args, **kwargs):
if not base_url:
raise ValueError("Service {} should have url provided".format(self.service.name))
url = '{}{}'.format(base_url.rstrip('/'), self.PATH)
rdata = requests.get(url)
rdata = requests.get(url, timeout=10)
if rdata.status_code != 200:
raise ValueError("Error response from api: ({}) {}".format(url, rdata))
data = rdata.json()['metrics']['metric']
Expand All @@ -206,7 +206,7 @@ def _collect(self, since, until, *args, **kwargs):
if not base_url:
raise ValueError("Service {} should have url provided".format(self.service.name))
url = '{}/monitoring/api/beacon/{}/'.format(base_url.rstrip('/'), self.service.service_type.name)
rdata = requests.get(url)
rdata = requests.get(url, timeout=10)
if rdata.status_code != 200:
raise ValueError("Error response from api: ({}) {}".format(url, rdata))
data = rdata.json()
Expand Down
41 changes: 24 additions & 17 deletions geonode/contrib/monitoring/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,23 @@
#########################################################################

import os
import re
import pytz
import threading
import traceback
import types
import Queue
import logging
import xmljson
import types
import re
from urllib import urlencode
from datetime import datetime, timedelta
from math import floor, ceil
import requests
import threading
import traceback

from xml.etree import ElementTree as etree
from math import floor, ceil
from urllib import urlencode
from urlparse import urlsplit
from bs4 import BeautifulSoup as bs
from requests.auth import HTTPBasicAuth
import requests
from datetime import datetime, timedelta
from xml.etree import ElementTree as etree

from django.conf import settings
from django.db.models.fields.related import RelatedField
Expand Down Expand Up @@ -116,11 +117,14 @@ def __init__(self, base_url):
self.base_url = base_url

def get_href(self, link, format=None):
href = link['href']
href = urlsplit(link['href'])
base_url = urlsplit(self.base_url)
if href and href.netloc != base_url.netloc:
href = href._replace(netloc=base_url.netloc)
if format is None:
return href
return href.geturl()
if format in self.REPORT_FORMATS:
href, ext = os.path.splitext(href)
href, ext = os.path.splitext(href.geturl())
return '{}.{}'.format(href, format)
return format

Expand All @@ -142,13 +146,13 @@ def get_requests(self, format=None, since=None, until=None):
print('checking', rest_url)
username = settings.OGC_SERVER['default']['USER']
password = settings.OGC_SERVER['default']['PASSWORD']
resp = requests.get(rest_url, auth=HTTPBasicAuth(username, password))
resp = requests.get(
rest_url,
auth=HTTPBasicAuth(username, password),
timeout=30)
doc = bs(resp.content)
links = doc.find_all('a')
for l in links:
# we're skipping this check, as gs can generate
# url with other base url
# if l.get('href').startswith(self.base_url):
href = self.get_href(l, format)
data = self.get_request(href, format=format)
if data:
Expand All @@ -159,7 +163,10 @@ def get_requests(self, format=None, since=None, until=None):
def get_request(self, href, format=format):
username = settings.OGC_SERVER['default']['USER']
password = settings.OGC_SERVER['default']['PASSWORD']
r = requests.get(href, auth=HTTPBasicAuth(username, password))
r = requests.get(
href,
auth=HTTPBasicAuth(username, password),
timeout=30)
if r.status_code != 200:
log.warning('Invalid response for %s: %s', href, r)
return
Expand Down
12 changes: 8 additions & 4 deletions geonode/security/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,8 @@ def get_geofence_rules_count():
headers = {'Content-type': 'application/json'}
r = requests.get(url + 'rest/geofence/rules/count.json',
headers=headers,
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code < 200 or r.status_code > 201):
logger.warning("Could not retrieve GeoFence Rules count.")

Expand Down Expand Up @@ -212,7 +213,8 @@ def get_highest_priority():
headers = {'Content-type': 'application/json'}
r = requests.get(url + 'rest/geofence/rules.json?page=' + str(rules_count - 1) + '&entries=1',
headers=headers,
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code < 200 or r.status_code > 201):
logger.warning("Could not retrieve GeoFence Rules count.")

Expand Down Expand Up @@ -243,7 +245,8 @@ def purge_geofence_all():
headers = {'Content-type': 'application/json'}
r = requests.get(url + 'rest/geofence/rules.json',
headers=headers,
auth=HTTPBasicAuth(user, passwd))
auth=HTTPBasicAuth(user, passwd),
timeout=10)
if (r.status_code < 200 or r.status_code > 201):
logger.warning("Could not Retrieve GeoFence Rules")
else:
Expand Down Expand Up @@ -287,7 +290,8 @@ def purge_geofence_layer_rules(resource):
"{}rest/geofence/rules.json?workspace={}&layer={}".format(
url, workspace, resource.layer.name),
headers=headers,
auth=HTTPBasicAuth(user, passwd)
auth=HTTPBasicAuth(user, passwd),
timeout=10
)
if (r.status_code >= 200 and r.status_code < 300):
gs_rules = r.json()
Expand Down
2 changes: 1 addition & 1 deletion geonode/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1539,7 +1539,7 @@
# add following lines to your local settings to enable monitoring
MONITORING_ENABLED = ast.literal_eval(os.environ.get('MONITORING_ENABLED', 'False'))
MONITORING_HOST_NAME = os.getenv("MONITORING_HOST_NAME", HOSTNAME)
MONITORING_SERVICE_NAME = 'geonode'
MONITORING_SERVICE_NAME = os.getenv("MONITORING_SERVICE_NAME", 'local-geoserver')

# how long monitoring data should be stored
MONITORING_DATA_TTL = timedelta(days=7)
Expand Down

0 comments on commit 1974350

Please sign in to comment.