Skip to content
This repository has been archived by the owner on Aug 27, 2021. It is now read-only.

Commit

Permalink
Security update and improvements in code (#12)
Browse files Browse the repository at this point in the history
  • Loading branch information
@flopezag @Cerfoglg
flopezag authored and Cerfoglg committed Feb 26, 2019
1 parent 13f2715 commit 6e2db7f
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 80 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
_build
.cache
.git

#IDE
.idea
16 changes: 13 additions & 3 deletions scripts/FIWARELabNodesSLA/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
[![License](https://img.shields.io/badge/license-Apache%20License,%20Version%202.0-green.svg)](http://www.apache.org/licenses/LICENSE-2.0)

- [General description](#general-description)
- [Requirements](#Requirements)
- [Deployment and run](#deployment-and-run)
- [Additional features](#additional-features)
- [Specify custom start date](#specify-custom-start-date)
Expand Down Expand Up @@ -55,7 +56,7 @@ a configuration file:
different from the default one
- `-s START_DAY`, `--start-day START_DAY`: sets the start date of the
computation (must be before the 2017-07-27). If not set, the
previous month is considered.
previous month is considered
- `-r REGION_ID`, `--region-id REGION_ID`: sets the region to analyse.
Setting "ALL", all regions will be analysed
- `-w`, `--weekend`: if specified, the week ends will be considered
Expand All @@ -78,15 +79,24 @@ a configuration file:
"Genoa", "Mexico", "Wroclaw", "Poznan", "SaoPaulo"]
```

## Requirements

The following software must be installed:

- Python 3.7
- pip
- virtualenv

## Deployment and run

The application is intended to be run both by each single region and centrally
by an admin for all the regions.

1. Create a virtualenv
1. Create a virtualenv and activate it

```bash
mkvirtualenv computingSLA
virtualenv -ppython3.7 computingSLA
source ./computingSLA/bin/activate
```

2. Install the dependencies
Expand Down
188 changes: 117 additions & 71 deletions scripts/FIWARELabNodesSLA/fiware_lab_nodes_sla.py
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
import argparse
import datetime
from distutils import util
from dateutil import parser
from dateutil.relativedelta import relativedelta
import os
import requests
import json
import threading
import time
import sys
from ConfigParser import ConfigParser
from dateutil.relativedelta import relativedelta
import pytz
from configparser import ConfigParser

class spinnerThread (threading.Thread):

class SpinnerThread (threading.Thread):

def __init__(self, text = None, delay=None):
def __init__(self, text=None, delay=None):
threading.Thread.__init__(self)
self.busy = True
self.spinner_generator = self.spinning_cursor()
Expand All @@ -22,10 +24,9 @@ def __init__(self, text = None, delay=None):
self.text = text

def run(self):

while 1:
if self.busy:
sys.stdout.write(self.text + self.spinner_generator.next())
sys.stdout.write(self.text + next(self.spinner_generator))
sys.stdout.flush()
time.sleep(0.1)
sys.stdout.write('\r')
Expand All @@ -35,30 +36,37 @@ def run(self):
sys.stdout.flush()
break

def spinning_cursor(self):
@staticmethod
def spinning_cursor():
while True:
for cursor in '|/-\\':
yield cursor

def stop(self):
self.busy = False

def computeSLAfromJson(region, json_data, end_date, weekend, sla, args):

def compute_sla_from_json(region, json_data, end_date, weekend, sla, args):
tot_counter = 0
fihealth_sum = 0.

undef_tot_number = 0

#loop over json returned to check FiHealthStatus->value_clean
for item in sorted(json_data['measures'], key=lambda x : x["timestamp"]):
#for item in json_data['measures']:
# loop over json returned to check FiHealthStatus->value_clean
for item in sorted(json_data['measures'], key=lambda x: x["timestamp"]):
# for item in json_data['measures']:
fihealth_value = item['FiHealthStatus']['value_clean']

timestamp = item['timestamp']
timestamp_date = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S")
timestamp = item['timestamp']
timestamp_date = parser.parse(timestamp)

if end_date.tzinfo is None or end_date.tzinfo.utcoffset(end_date) is None:
end_date = pytz.UTC.localize(end_date)

# timestamp_date = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S")
if timestamp_date <= end_date:
timestamp_weekday = timestamp_date.weekday()
if(weekend or (timestamp_weekday != 5 and timestamp_weekday != 6)):
if weekend or (timestamp_weekday != 5 and timestamp_weekday != 6):
tot_counter += 1

value_to_print = "undefined"
Expand All @@ -70,26 +78,29 @@ def computeSLAfromJson(region, json_data, end_date, weekend, sla, args):
undef_tot_number += 1

if args.log:
sys.stdout.write(timestamp+": "+ str(value_to_print) + "\n")
sys.stdout.write(timestamp+": " + str(value_to_print) + "\n")
else:
if args.log:
sys.stdout.write(timestamp+": "+"not considered\n")
sys.stdout.write(timestamp+": " + "not considered\n")
if args.log:
sys.stdout.write("Day of the week: "+str(timestamp_weekday) + "\n")
sys.stdout.write("Day of the week: " + str(timestamp_weekday) + "\n")
sys.stdout.write("-------------\n")

avg_percentage = 0

if tot_counter>0:
if tot_counter > 0:

if (tot_counter - undef_tot_number) > 0:
avg_percentage = fihealth_sum/(tot_counter - undef_tot_number)
avg_percentage = fihealth_sum / (tot_counter - undef_tot_number)

sla_value = str(round(avg_percentage*100,2))+"%"
sla_value = str(round(avg_percentage*100, 2)) + "%"

if undef_tot_number > (tot_counter*undef_percentage/100):

sys.stdout.write("The SLA for "+region+" is not respected: too many undefined ( > "+str(undef_percentage)+"% ) "+str(undef_tot_number)+"/"+str(tot_counter)+" ( partial SLA value: "+sla_value+" )"+"\n")
sys.stdout.write("The SLA for " + region + " is not respected: too many undefined "
"( > "+str(undef_percentage)+"% ) " + str(undef_tot_number) + "/"
+ str(tot_counter) + " ( partial SLA value: "+sla_value+" )" + "\n")

sys.stdout.flush()
return

Expand All @@ -103,6 +114,7 @@ def computeSLAfromJson(region, json_data, end_date, weekend, sla, args):
else:
sys.stdout.write("The SLA for "+region+" is not respected: "+sla_value+"\n")


def main():

global script_version
Expand All @@ -115,7 +127,7 @@ def main():
args = arg_parser()

if args.version:
sys.stdout.write("SLA script "+script_version+"\n")
sys.stdout.write("SLA script " + script_version+"\n")
sys.exit(-1)

sys.stdout.write("\n######################################\n")
Expand All @@ -132,47 +144,47 @@ def main():
if args.start_day is None:
# Setup default temporal period in which to work
# TODO: sys.stdout.write on logger we are using default dates
#end_date = datetime.datetime.now()
#end_string = (end_date).strftime("%Y-%m-%d")
# end_date = datetime.datetime.now()
# end_string = (end_date).strftime("%Y-%m-%d")

#start_date = datetime.datetime.strptime(end_date,"%Y-%m-%d") - relativedelta(months=1)
#start_string = (start_date).strftime("%Y-%m-%d")
#TODO consider last entire month
# start_date = datetime.datetime.strptime(end_date,"%Y-%m-%d") - relativedelta(months=1)
# start_string = (start_date).strftime("%Y-%m-%d")
# TODO consider last entire month

#today = datetime.datetime.now()
#d = today - relativedelta(months=1)
#start_date = date(d.year, d.month, 1)
#datetime.date(2008, 12, 1)
#date(today.year, today.month, 1) - relativedelta(days=1)
#datetime.date(2008, 12, 31)
# today = datetime.datetime.now()
# d = today - relativedelta(months=1)
# start_date = date(d.year, d.month, 1)
# datetime.date(2008, 12, 1)
# date(today.year, today.month, 1) - relativedelta(days=1)
# datetime.date(2008, 12, 31)

today = datetime.datetime.now()
d = today - relativedelta(months=1)
start_date = d.replace(day=1)
start_string = (start_date).strftime("%Y-%m-%d")
start_string = start_date.strftime("%Y-%m-%d")
end_date = today.replace(day=1) - relativedelta(days=1)
end_string = (end_date).strftime("%Y-%m-%d")
end_string = end_date.strftime("%Y-%m-%d")

else:
# Setup user temporal period in which to work
# TODO: sys.stdout.write on logger we are using user dates
start_string = args.start_day
try:
start_date = datetime.datetime.strptime(start_string,"%Y-%m-%d")
start_date = datetime.datetime.strptime(start_string, "%Y-%m-%d")
except Exception:
sys.stdout.write("The start date is not valid. Please use the format '2017-06-27'")
sys.stdout.write("The start date is not valid. Please use the format '2017-06-27'\n")
sys.exit(-1)

#end_string = "2017-07-27"
end_date = datetime.datetime.now() - relativedelta(days=1)
end_string = (end_date).strftime("%Y-%m-%d")
#end_date = datetime.datetime.strptime(end_string,"%Y-%m-%d")
# end_string = "2017-07-27"
# end_date = datetime.datetime.now() - relativedelta(days=1)
end_date = pytz.utc.localize(datetime.datetime.utcnow()) - relativedelta(days=1)
end_string = end_date.strftime("%Y-%m-%d")
# end_date = datetime.datetime.strptime(end_string,"%Y-%m-%d")

if start_date >= end_date:
sys.stdout.write("The start date is not valid. Please specify an older date")
sys.exit(-1)



# Read config file
if not os.path.isfile(config_file):
sys.stdout.write("Configuration file not found: {}").format(config_file)
Expand All @@ -191,19 +203,19 @@ def main():

# Get region to work on
if args.region_id is None:
regions = [config.get('region','id')]
elif (args.region_id).lower() == "all":
regions = json.loads(config.get("all","list"))
regions = [config.get('region', 'id')]
elif args.region_id.lower() == "all":
regions = json.loads(config.get("all", "list"))
sys.stdout.write("# The regions are: " + ", ".join(regions) + "\n")
else:
regions = [args.region_id]
sys.stdout.write("# The region is: " + regions[0] + "\n")

# Get sla parameter to work with
try:
sla = float(config.get('sla','value'))
except Exception as e:
sys.stdout.write("The sla value in the config file is not valid. Please specify a valid sla value: 0.95")
sla = config.getfloat('sla', 'value')
except Exception:
sys.stdout.write("The sla value in the config file is not valid. Please specify a valid sla value: 0.95\n")
sys.exit(-1)

sys.stdout.write("# The SLA parameter is: " + str(sla*100) + "%\n")
Expand All @@ -212,51 +224,85 @@ def main():
if not args.weekend:
weekend = False
else:
weekend = True#util.strtobool(args.weekend)
weekend = True # util.strtobool(args.weekend)

considered = "No"
if weekend:
considered = "Yes"

sys.stdout.write("# The week ends will be considered: " + considered + "\n")
sys.stdout.write("#\n")
sys.stdout.write("######################################\n")
sys.stdout.flush()

for region_id in regions:
#perform request
url = config.get('monitoring','url')+'/monitoring/regions/'+region_id+'/services'
spinnerThread1 = spinnerThread("Calling " + url + "... ", 0.1)
# perform request
url = config.get('monitoring', 'url') + '/monitoring/regions/' + region_id + '/services'
spinner_thread1 = SpinnerThread("Calling " + url + "... ", 0.1)
sys.stdout.write("\n")
sys.stdout.flush()
spinnerThread1.start()
spinner_thread1.start()

payload = {'since': start_string, 'aggregate': 'd'}
print(repr(url))
print(url)
response = requests.get(url, params=payload)
#check json returned
spinnerThread1.stop()

# check json returned
spinner_thread1.stop()
time.sleep(0.1)
sys.stdout.write("\n\n")
if(response.status_code == 200):
if response.status_code == 200:
json_data = response.json()
#sys.stdout.write(json_data)
computeSLAfromJson(region_id, json_data, end_date, weekend, sla, args)
# sys.stdout.write(json_data)
compute_sla_from_json(region_id, json_data, end_date, weekend, sla, args)
else:
sys.stdout.write("Response status code: " + str(response.status_code) + "\n")

#sys.stdout.write("# REGIONS: " + ", ".join(regions) + "\n")
# sys.stdout.write("# REGIONS: " + ", ".join(regions) + "\n")
sys.stdout.flush()



# Argument management
def arg_parser():
parser = argparse.ArgumentParser(description='SLA computing')
parser.add_argument("-c", "--config-file", help="-c Config file", required=False)
parser.add_argument("-s", "--start-day", help="-s 2017-06-27", required=False)
parser.add_argument("-r", "--region-id", help="-r Spain2", required=False)
parser.add_argument("-w", "--weekend", help="if specified, the week ends will be considered", required=False, action='store_true')
parser.add_argument("-l", "--log", help="if specified, prints the SLA for each day", required=False, action='store_true')
parser.add_argument("-v", "--version", help="if specified, prints the version of the SLA script", required=False, action='store_true')
return parser.parse_args()
parser_arg = argparse.ArgumentParser(description='SLA computing')

parser_arg.add_argument("-c",
"--config-file",
help="-c Config file",
required=False)

parser_arg.add_argument("-s",
"--start-day",
help="-s 2017-06-27",
required=False)

parser_arg.add_argument("-r",
"--region-id",
help="-r Spain2",
required=False)

parser_arg.add_argument("-w",
"--weekend",
help="if specified, the week ends will be considered",
required=False,
action='store_true')

parser_arg.add_argument("-l",
"--log",
help="if specified, prints the SLA for each day",
required=False,
action='store_true')

parser_arg.add_argument("-v",
"--version",
help="if specified, prints the version of the SLA script",
required=False,
action='store_true')

return parser_arg.parse_args()


if __name__ == '__main__':
main()
main()
print()

0 comments on commit 6e2db7f

Please sign in to comment.