Skip to content
Permalink
Browse files

feat: quads reporting

added reporting functionality.
added build_delta to store the time delta taken by M&R.
added report library with reporting functionality which can be
called from quads-cli with the optional argument `--report`.
additionally removed logic on M&R to not set foreman user and pass
for cloud01 hosts.

Change-Id: If68a9c85c4e09de592c9622d552c19f01ab0fc51
  • Loading branch information
grafuls committed Nov 8, 2019
1 parent 52b9090 commit e18daaadddeb9ebdc05f83bf8238d976fe6296a6
Showing with 120 additions and 8 deletions.
  1. +8 −0 bin/quads-cli
  2. +6 −3 quads/model.py
  3. +12 −5 quads/tools/move_and_rebuild_hosts.py
  4. +94 −0 quads/tools/report.py
@@ -31,6 +31,7 @@ import os
from json import JSONDecodeError
from quads.config import conf, API_URL
from quads.quads import Api as QuadsApi
from quads.tools import report
from quads.tools.move_and_rebuild_hosts import move_and_rebuild, switch_config
from quads.model import Cloud, Schedule, Host

@@ -246,6 +247,11 @@ def main(_args):

exit(0)

elif _args.action == "report":
report.report(logger)

exit(0)

# Cloud Add / Modify
elif _args.cloudresource:
data = {'name': _args.cloudresource,
@@ -595,6 +601,8 @@ if __name__ == '__main__':
help='List the host interfaces')
group.add_argument('--find-free-cloud', dest='action', action='store_const', const='free_cloud',
help='List available hosts on a specific time frame')
group.add_argument('--report', dest='action', action='store_const', const='report',
help='Quads reporting')
parser.add_argument('--host', dest='host', type=str, default=None, help='Specify the host to query')
parser.add_argument('--cloud-only', dest='cloudonly', type=str, default=None,
help='Limit full report to hosts only in this cloud')
@@ -1,5 +1,5 @@
import ipaddress
from datetime import datetime
import datetime
from mongoengine import (
connect,
Document,
@@ -16,6 +16,7 @@
LongField,
EmbeddedDocumentField,
)

from quads.helpers import param_check

import os
@@ -215,6 +216,8 @@ class Schedule(Document):
host = ReferenceField(Host, required=True)
start = DateTimeField()
end = DateTimeField()
build_start = DateTimeField()
build_end = DateTimeField()
index = IntField()
meta = {'strict': False}

@@ -260,14 +263,14 @@ def is_host_available(self, queryset, host, start, end, exclude=None):

@queryset_manager
def future_schedules(self, queryset, host):
now = datetime.now()
now = datetime.datetime.now()
_query = Q(host=host) & Q(end__gte=now)
return queryset.filter(_query)

@queryset_manager
def current_schedule(self, queryset, date=None, host=None, cloud=None):
if not date:
date = datetime.now()
date = datetime.datetime.now()
_query = Q(start__lte=date) & Q(end__gte=date)
if host:
_query = _query & Q(host=host)
@@ -7,7 +7,7 @@

from quads.config import conf
from quads.helpers import is_supported, is_supermicro, get_vlan
from quads.model import Host, Cloud
from quads.model import Host, Cloud, Schedule
from quads.tools.badfish import badfish_factory, BadfishException
from quads.tools.foreman import Foreman
from quads.tools.juniper_convert_port_public import juniper_convert_port_public
@@ -113,6 +113,7 @@ def switch_config(host, old_cloud, new_cloud):


async def move_and_rebuild(host, old_cloud, new_cloud, semaphore, rebuild=False, loop=None):
build_start = datetime.now()
logger.debug("Moving and rebuilding host: %s" % host)

untouchable_hosts = conf["untouchable_hosts"]
@@ -141,11 +142,12 @@ def switch_config(host, old_cloud, new_cloud):
remove_result = await foreman.remove_role(_old_cloud_obj.name, _host_obj.name)
foreman_results.append(remove_result)

add_result = await foreman.add_role(_new_cloud_obj.name, _host_obj.name)
foreman_results.append(add_result)
if _new_cloud_obj.name != "cloud01":
add_result = await foreman.add_role(_new_cloud_obj.name, _host_obj.name)
foreman_results.append(add_result)

update_result = await foreman.update_user_password(_new_cloud_obj.name, ipmi_new_pass)
foreman_results.append(update_result)
update_result = await foreman.update_user_password(_new_cloud_obj.name, ipmi_new_pass)
foreman_results.append(update_result)

for result in foreman_results:
if isinstance(result, Exception) or not result:
@@ -237,6 +239,11 @@ def switch_config(host, old_cloud, new_cloud):
logger.debug(ex)
logger.error(f"There was something wrong resetting IPMI on {host}.")

schedule = Schedule.current_schedule(cloud=_new_cloud_obj, host=_host_obj).first()
if schedule:
schedule.update(build_start=build_start, build_end=datetime.now())
schedule.save()

logger.debug("Updating host: %s")
_host_obj.update(cloud=_new_cloud_obj, build=False, last_build=datetime.now())
return True
@@ -0,0 +1,94 @@
import calendar
import logging
import sys

from quads.model import Schedule, Host
from datetime import datetime, date, timedelta

logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.propagate = False
logging.basicConfig(level=logging.INFO, format="%(message)s")


def report(_logger):
now = datetime.now()
now = now.replace(hour=22, minute=0, second=0)
month = now.month
days = calendar.mdays[month]
hosts = Host.objects()
next_sunday = now + timedelta(days=(6 - now.weekday()))

_logger.info(f"Quads report for {now.year}-{month}:")

total_allocated_month = 0
total_hosts = len(hosts)
for day in range(1, days + 1):
schedules = Schedule.current_schedule(date=date(now.year, month, day)).count()
total_allocated_month += schedules
utilized = total_allocated_month * 100 // (total_hosts * days)
_logger.info(f"Percentage Utilized: {utilized}%")

schedules = Schedule.objects(build_start__ne=None, build_end__ne=None)
total = timedelta()
for schedule in schedules:
total += schedule.build_end - schedule.build_start
if schedules:
average_build = total / len(schedules)
logger.info(f"Average build delta: {average_build}")

hosts_summary = {}
for host in hosts:
host_type = host.name.split(".")[0].split("-")[-1]
if not hosts_summary.get(host_type):
hosts_summary[host_type] = []
hosts_summary[host_type].append(host)

headers = ["Server Type", "Total", "Free", "Scheduled", "2 weeks", "4 weeks"]
logger.info(
f"{headers[0]:<12}| "
f"{headers[1]:>5}| "
f"{headers[2]:>5}| "
f"{headers[3]:>9}| "
f"{headers[4]:>7}| "
f"{headers[5]:>7}"
)
for host_type, _hosts in hosts_summary.items():
scheduled_count = 0
two_weeks_availability_count = 0
four_weeks_availability_count = 0
for host in _hosts:
schedule = Schedule.current_schedule(host=host)
if schedule:
scheduled_count += 1

two_weeks_availability = Schedule.is_host_available(
host=host.name,
start=next_sunday,
end=next_sunday + timedelta(weeks=2)
)
if two_weeks_availability:
two_weeks_availability_count += 1

four_weeks_availability = Schedule.is_host_available(
host=host.name,
start=next_sunday,
end=next_sunday + timedelta(weeks=4)
)
if four_weeks_availability:
four_weeks_availability_count += 1

free = len(_hosts) - scheduled_count
schedule_percent = scheduled_count * 100 // len(_hosts)
logger.info(
f"{host_type:<12}| "
f"{len(_hosts):>5}| "
f"{free:>5}| "
f"{schedule_percent:>8}%| "
f"{two_weeks_availability_count:>7}| "
f"{four_weeks_availability_count:>7}"
)


if __name__ == "__main__":
report(logger)

0 comments on commit e18daaa

Please sign in to comment.
You can’t perform that action at this time.