From 1bf446b27d5b525ecf3a40f3467859fb357a65f4 Mon Sep 17 00:00:00 2001 From: Dan Trickey Date: Wed, 27 Dec 2023 22:18:40 +0000 Subject: [PATCH 1/6] Implementation of netbox integration --- kmicms/integrations/netbox/__init__.py | 7 + kmicms/integrations/netbox/client.py | 60 ++++++++ kmicms/integrations/netbox/exceptions.py | 2 + kmicms/integrations/netbox/graphql.py | 62 +++++++++ kmicms/integrations/netbox/queries.py | 128 ++++++++++++++++++ kmicms/kmicms/configuration.dev.py | 5 + kmicms/kmicms/configuration.example.py | 7 + kmicms/kmicms/configuration.prod.py | 5 + kmicms/kmicms/settings.py | 9 +- kmicms/pages/home/models.py | 2 +- kmicms/pages/infra/__init__.py | 0 kmicms/pages/infra/apps.py | 6 + kmicms/pages/infra/migrations/0001_initial.py | 43 ++++++ kmicms/pages/infra/migrations/__init__.py | 0 kmicms/pages/infra/models.py | 81 +++++++++++ .../infra/inc/breadcrumbs_subpage.html | 28 ++++ .../templates/infra/inc/colored_badge.html | 3 + .../templates/infra/inc/interface_table.html | 29 ++++ .../infra/templates/infra/inc/server_dl.html | 27 ++++ .../templates/infra/netbox_device_index.html | 38 ++++++ .../templates/infra/netbox_device_view.html | 32 +++++ .../infra/templates/infra/netbox_error.html | 14 ++ .../infra/netbox_infrastructure_page.html | 28 ++++ .../templates/infra/netbox_vm_index.html | 38 ++++++ .../infra/templates/infra/netbox_vm_view.html | 34 +++++ kmicms/pages/standard_page/models.py | 2 +- kmicms/templates/components/breadcrumbs.html | 20 +++ 27 files changed, 707 insertions(+), 3 deletions(-) create mode 100644 kmicms/integrations/netbox/__init__.py create mode 100644 kmicms/integrations/netbox/client.py create mode 100644 kmicms/integrations/netbox/exceptions.py create mode 100644 kmicms/integrations/netbox/graphql.py create mode 100644 kmicms/integrations/netbox/queries.py create mode 100644 kmicms/pages/infra/__init__.py create mode 100644 kmicms/pages/infra/apps.py create mode 100644 kmicms/pages/infra/migrations/0001_initial.py create mode 100644 kmicms/pages/infra/migrations/__init__.py create mode 100644 kmicms/pages/infra/models.py create mode 100644 kmicms/pages/infra/templates/infra/inc/breadcrumbs_subpage.html create mode 100644 kmicms/pages/infra/templates/infra/inc/colored_badge.html create mode 100644 kmicms/pages/infra/templates/infra/inc/interface_table.html create mode 100644 kmicms/pages/infra/templates/infra/inc/server_dl.html create mode 100644 kmicms/pages/infra/templates/infra/netbox_device_index.html create mode 100644 kmicms/pages/infra/templates/infra/netbox_device_view.html create mode 100644 kmicms/pages/infra/templates/infra/netbox_error.html create mode 100644 kmicms/pages/infra/templates/infra/netbox_infrastructure_page.html create mode 100644 kmicms/pages/infra/templates/infra/netbox_vm_index.html create mode 100644 kmicms/pages/infra/templates/infra/netbox_vm_view.html create mode 100644 kmicms/templates/components/breadcrumbs.html diff --git a/kmicms/integrations/netbox/__init__.py b/kmicms/integrations/netbox/__init__.py new file mode 100644 index 0000000..6ed59b4 --- /dev/null +++ b/kmicms/integrations/netbox/__init__.py @@ -0,0 +1,7 @@ +from .client import NetboxClient +from .exceptions import NetboxRequestError + +__all__ = [ + "NetboxClient", + "NetboxRequestError", +] diff --git a/kmicms/integrations/netbox/client.py b/kmicms/integrations/netbox/client.py new file mode 100644 index 0000000..c23895b --- /dev/null +++ b/kmicms/integrations/netbox/client.py @@ -0,0 +1,60 @@ +from typing import Any + +from django.conf import settings +from django.core.cache import cache + +from .graphql import netbox_query +from .queries import GET_DEVICE_QUERY, GET_VM_QUERY, LIST_DEVICE_QUERY, LIST_VM_QUERY + + +class NetboxClient: + def __init__(self, *, cache_ttl_seconds: int = settings.NETBOX_CACHE_TTL) -> None: + self.cache_ttl_seconds = cache_ttl_seconds + + def get_device(self, device_id: int) -> dict[str, Any]: + cache_key = f"netbox__device__{device_id}" + + if cache_value := cache.get(cache_key): + return cache_value + + device = netbox_query(GET_DEVICE_QUERY, {"deviceId": device_id})["device"] + cache.set(cache_key, device, timeout=self.cache_ttl_seconds) + + return device + + def get_vm(self, vm_id: int) -> dict[str, Any]: + cache_key = f"netbox__vm__{vm_id}" + + if cache_value := cache.get(cache_key): + return cache_value + + vm = netbox_query(GET_VM_QUERY, {"VMId": vm_id})["virtual_machine"] + cache.set(cache_key, vm, timeout=self.cache_ttl_seconds) + + return vm + + def list_devices( + self, + ) -> dict[str, Any]: + cache_key = "netbox__device_list" + + if cache_value := cache.get(cache_key): + return cache_value + + device = netbox_query(LIST_DEVICE_QUERY)["device_list"] + cache.set(cache_key, device, timeout=self.cache_ttl_seconds) + + return device + + def list_vms( + self, + ) -> dict[str, Any]: + cache_key = "netbox__vm_list" + + if cache_value := cache.get(cache_key): + return cache_value + + vms = netbox_query(LIST_VM_QUERY)["virtual_machine_list"] + cache.set(cache_key, vms, timeout=self.cache_ttl_seconds) + + return vms diff --git a/kmicms/integrations/netbox/exceptions.py b/kmicms/integrations/netbox/exceptions.py new file mode 100644 index 0000000..00f59b6 --- /dev/null +++ b/kmicms/integrations/netbox/exceptions.py @@ -0,0 +1,2 @@ +class NetboxRequestError(Exception): + """An error occurred in a request to Netbox.""" diff --git a/kmicms/integrations/netbox/graphql.py b/kmicms/integrations/netbox/graphql.py new file mode 100644 index 0000000..ba26809 --- /dev/null +++ b/kmicms/integrations/netbox/graphql.py @@ -0,0 +1,62 @@ +import json +from typing import Any + +import requests +from django.conf import settings + +from .exceptions import NetboxRequestError + + +def netbox_query(query: str, variables: dict[str, str] | None = None) -> dict[str, Any]: # noqa: FA102 + payload = { + "query": query, + } + + if variables is not None: + payload["variables"] = variables + + try: + resp = requests.post( + settings.NETBOX_GRAPHQL_ENDPOINT, + headers={ + "Authorization": f"Token {settings.NETBOX_API_TOKEN}", + "Accept": "application/json", + "Content-Type": "application/json", + }, + json=payload, + timeout=settings.NETBOX_REQUEST_TIMEOUT, + ) + except ConnectionError as e: + raise NetboxRequestError("Unable to connect to netbox") from e + except requests.RequestException as e: + raise NetboxRequestError("Error when requesting data from netbox") from e + + try: + resp.raise_for_status() + except requests.RequestException as e: + # Include GraphQL errors in the exception message. + try: + data = resp.json() + except json.JSONDecodeError: + message = str(e) + else: + errors = data.get("errors") + message = f"{e}: GraphQL errors: {errors}" + raise NetboxRequestError(message) from e + + try: + gql_response = resp.json() + except json.JSONDecodeError as e: + raise NetboxRequestError("Netbox returned invalid JSON") from e + + # Check for and raise any GraphQL errors from successful responses. + if "errors" in gql_response: + errors = gql_response["errors"] + raise NetboxRequestError(f"Invalid GraphQL response: {errors}") + + try: + return gql_response["data"] + except KeyError as e: + raise NetboxRequestError( + "Netbox API response did not contain data.", + ) from e diff --git a/kmicms/integrations/netbox/queries.py b/kmicms/integrations/netbox/queries.py new file mode 100644 index 0000000..16de02c --- /dev/null +++ b/kmicms/integrations/netbox/queries.py @@ -0,0 +1,128 @@ +GET_DEVICE_QUERY = """ +query DeviceInfo($deviceId: Int!) { + device(id: $deviceId) { + name + role { + name + color + } + description + comments + platform { + name + } + interfaces { + description + name + enabled + ip_addresses { + display + dns_name + } + } + device_type { + model + manufacturer { + name + } + front_image + rear_image + } + tags { + name + color + } + rack { + name + } + location { + name + } + position + status + } +} +""" + +LIST_DEVICE_QUERY = """ +query listDevices { + device_list { + id + name + role { + name + color + } + platform { + name + } + device_type { + model + manufacturer { + name + } + } + rack { + name + } + status + } +} +""" + +GET_VM_QUERY = """ +query VMInfo($VMId: Int!) { + virtual_machine(id: $VMId) { + name + cluster { + name + } + role { + name + color + } + description + comments + platform { + name + } + interfaces { + description + name + enabled + ip_addresses { + display + dns_name + } + } + vcpus + memory + disk + tags { + name + color + } + status + } +} +""" + +LIST_VM_QUERY = """ +query listVMs { + virtual_machine_list { + id + name + role { + name + color + } + platform { + name + } + cluster { + name + } + status + } +} +""" diff --git a/kmicms/kmicms/configuration.dev.py b/kmicms/kmicms/configuration.dev.py index 08279c3..6a43a6a 100644 --- a/kmicms/kmicms/configuration.dev.py +++ b/kmicms/kmicms/configuration.dev.py @@ -31,3 +31,8 @@ RECAPTCHA_PUBLIC_KEY = "" RECAPTCHA_PRIVATE_KEY = "" + +NETBOX_GRAPHQL_ENDPOINT = "https://netbox.example.com/graphql/" +NETBOX_API_TOKEN = "abc" # noqa: S105 +NETBOX_REQUEST_TIMEOUT = 0.5 +NETBOX_CACHE_TTL = 300 diff --git a/kmicms/kmicms/configuration.example.py b/kmicms/kmicms/configuration.example.py index f94f73a..84d822d 100644 --- a/kmicms/kmicms/configuration.example.py +++ b/kmicms/kmicms/configuration.example.py @@ -75,3 +75,10 @@ # Time zone (default: UTC) TIME_ZONE = "UTC" + +# Netbox + +NETBOX_GRAPHQL_ENDPOINT = "https://netbox.example.com/graphql/" +NETBOX_API_TOKEN = "abc" # noqa: S105 +NETBOX_REQUEST_TIMEOUT = 0.5 +NETBOX_CACHE_TTL = 300 diff --git a/kmicms/kmicms/configuration.prod.py b/kmicms/kmicms/configuration.prod.py index 9d4e106..f0266c4 100644 --- a/kmicms/kmicms/configuration.prod.py +++ b/kmicms/kmicms/configuration.prod.py @@ -46,3 +46,8 @@ TIME_ZONE = "Europe/London" WAGTAILADMIN_BASE_URL = "https://kmicms.containers-1.sown.org.uk" + +NETBOX_GRAPHQL_ENDPOINT = "https://netbox.sown.org.uk/graphql/" +NETBOX_API_TOKEN = os.environ.get("NETBOX_API_TOKEN") +NETBOX_REQUEST_TIMEOUT = 0.5 +NETBOX_CACHE_TTL = 300 diff --git a/kmicms/kmicms/settings.py b/kmicms/kmicms/settings.py index 2d6e6f8..df824d7 100644 --- a/kmicms/kmicms/settings.py +++ b/kmicms/kmicms/settings.py @@ -92,6 +92,7 @@ "core", "pages.contact", "pages.home", + "pages.infra", "pages.standard_page", # 3rd party "compressor", @@ -101,6 +102,7 @@ # Wagtail / Django "wagtail.contrib.forms", "wagtail.contrib.redirects", + "wagtail.contrib.routable_page", "wagtail.contrib.settings", "wagtail.embeds", "wagtail.sites", @@ -313,6 +315,11 @@ CRISPY_TEMPLATE_PACK = "bootstrap5" # ReCAPTCHA - RECAPTCHA_PUBLIC_KEY = getattr(configuration, "RECAPTCHA_PUBLIC_KEY", None) RECAPTCHA_PRIVATE_KEY = getattr(configuration, "RECAPTCHA_PRIVATE_KEY", None) + +# Netbox Integration +NETBOX_GRAPHQL_ENDPOINT = getattr(configuration, "NETBOX_GRAPHQL_ENDPOINT") +NETBOX_API_TOKEN = getattr(configuration, "NETBOX_API_TOKEN") +NETBOX_REQUEST_TIMEOUT = getattr(configuration, "NETBOX_REQUEST_TIMEOUT", 0.5) +NETBOX_CACHE_TTL = getattr(configuration, "NETBOX_CACHE_TTL", 300) diff --git a/kmicms/pages/home/models.py b/kmicms/pages/home/models.py index 03adbce..4f18f69 100644 --- a/kmicms/pages/home/models.py +++ b/kmicms/pages/home/models.py @@ -8,7 +8,7 @@ class HomePage(Page): parent_page_types = ["wagtailcore.Page"] - subpage_types = ["standard_page.StandardPage", "contact.ContactFormPage"] + subpage_types = ["standard_page.StandardPage", "contact.ContactFormPage", "infra.NetboxInfrastructurePage"] content = StreamField(BodyBlock(), use_json_field=True) diff --git a/kmicms/pages/infra/__init__.py b/kmicms/pages/infra/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kmicms/pages/infra/apps.py b/kmicms/pages/infra/apps.py new file mode 100644 index 0000000..c1981aa --- /dev/null +++ b/kmicms/pages/infra/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class InfraConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "pages.infra" diff --git a/kmicms/pages/infra/migrations/0001_initial.py b/kmicms/pages/infra/migrations/0001_initial.py new file mode 100644 index 0000000..48af342 --- /dev/null +++ b/kmicms/pages/infra/migrations/0001_initial.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.7 on 2023-12-27 21:44 + +import django.db.models.deletion +import wagtail.contrib.routable_page.models +import wagtail.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + ("wagtailcore", "0089_log_entry_data_json_null_to_object"), + ] + + operations = [ + migrations.CreateModel( + name="NetboxInfrastructurePage", + fields=[ + ( + "page_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="wagtailcore.page", + ), + ), + ("introduction", wagtail.fields.RichTextField()), + ("device_description", wagtail.fields.RichTextField()), + ("vm_description", wagtail.fields.RichTextField()), + ], + options={ + "abstract": False, + }, + bases=( + wagtail.contrib.routable_page.models.RoutablePageMixin, + "wagtailcore.page", + ), + ), + ] diff --git a/kmicms/pages/infra/migrations/__init__.py b/kmicms/pages/infra/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kmicms/pages/infra/models.py b/kmicms/pages/infra/models.py new file mode 100644 index 0000000..8ebc44b --- /dev/null +++ b/kmicms/pages/infra/models.py @@ -0,0 +1,81 @@ +from django.http import Http404, HttpRequest, HttpResponse +from wagtail.admin.panels import FieldPanel, TitleFieldPanel +from wagtail.contrib.routable_page.models import RoutablePageMixin, path +from wagtail.fields import RichTextField +from wagtail.models import Page + +from integrations.netbox import NetboxClient, NetboxRequestError + + +class NetboxInfrastructurePage(RoutablePageMixin, Page): + max_count = 1 + subpage_types = [] + + introduction = RichTextField() + device_description = RichTextField() + vm_description = RichTextField() + + content_panels = [ + TitleFieldPanel("title"), + FieldPanel("introduction"), + FieldPanel("device_description"), + FieldPanel("vm_description"), + ] + + def _get_client(self) -> NetboxClient: + return NetboxClient() + + def _handle_error(self, request: HttpRequest) -> HttpResponse: + return self.render( + request, + template="infra/netbox_error.html", + ) + + @path("devices/", name="device_index") + def device_index(self, request: HttpRequest) -> HttpResponse: + try: + client = self._get_client() + devices = client.list_devices() + except NetboxRequestError: + return self._handle_error(request) + + return self.render( + request, + context_overrides={"devices": devices}, + template="infra/netbox_device_index.html", + ) + + @path("devices//", name="device_view") + def device_info(self, request: HttpRequest, *, device_id: int) -> HttpResponse: + try: + client = self._get_client() + device = client.get_device(device_id) + except NetboxRequestError: + return self._handle_error(request) + + if device is None: + raise Http404() + + return self.render(request, context_overrides={"device": device}, template="infra/netbox_device_view.html") + + @path("vm/", name="vm_index") + def vm_index(self, request: HttpRequest) -> HttpResponse: + try: + client = self._get_client() + vms = client.list_vms() + except NetboxRequestError: + return self._handle_error(request) + return self.render(request, context_overrides={"vms": vms}, template="infra/netbox_vm_index.html") + + @path("vm//", name="vm_view") + def vm_info(self, request: HttpRequest, *, vm_id: int) -> HttpResponse: + try: + client = self._get_client() + vm = client.get_vm(vm_id) + except NetboxRequestError: + return self._handle_error(request) + + if vm is None: + raise Http404() + + return self.render(request, context_overrides={"vm": vm}, template="infra/netbox_vm_view.html") diff --git a/kmicms/pages/infra/templates/infra/inc/breadcrumbs_subpage.html b/kmicms/pages/infra/templates/infra/inc/breadcrumbs_subpage.html new file mode 100644 index 0000000..d95fd66 --- /dev/null +++ b/kmicms/pages/infra/templates/infra/inc/breadcrumbs_subpage.html @@ -0,0 +1,28 @@ +{% load wagtailcore_tags wagtailroutablepage_tags %} + +{% block breadcrumbs %} + {% if page.get_ancestors|length > 1 %} + + {% endif %} +{% endblock breadcrumbs %} \ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/inc/colored_badge.html b/kmicms/pages/infra/templates/infra/inc/colored_badge.html new file mode 100644 index 0000000..975aafe --- /dev/null +++ b/kmicms/pages/infra/templates/infra/inc/colored_badge.html @@ -0,0 +1,3 @@ + + {{ info.name }} + \ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/inc/interface_table.html b/kmicms/pages/infra/templates/infra/inc/interface_table.html new file mode 100644 index 0000000..3e586d3 --- /dev/null +++ b/kmicms/pages/infra/templates/infra/inc/interface_table.html @@ -0,0 +1,29 @@ +{% if interfaces %} +

Interfaces

+ + + + + + + + + + + {% for interface in interfaces %} + + + + + + + {% endfor %} + +
NameEnabledDescriptionIP Addresses
{{ interface.name }}{% if interface.enabled %}Enabled{% else %}Disabled{% endif %}{% if interface.description %}{{ interface.description}}{% else %}-{% endif %} +
    + {% for address in interface.ip_addresses %} +
  • {{ address.display }}: {{ address.dns_name }}
    • + {% endfor %} +
+
+{% endif %} \ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/inc/server_dl.html b/kmicms/pages/infra/templates/infra/inc/server_dl.html new file mode 100644 index 0000000..b96c14e --- /dev/null +++ b/kmicms/pages/infra/templates/infra/inc/server_dl.html @@ -0,0 +1,27 @@ +
+
Status
+
{{ server.status }}
+ +
Role
+
{% include 'infra/inc/colored_badge.html' with info=server.role %}
+ +
Platform
+
{{ server.platform.name }}
+ +
Tags
+
+ {% for tag in server.tags %} + {% include "infra/inc/colored_badge.html" with info=tag %} + {% endfor %} +
+ + {% if server.description %} +
Description
+
{{ server.description }}
+ {% endif %} + + {% if server.comments %} +
Comments
+
{{ server.comments }}
+ {% endif %} +
\ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/netbox_device_index.html b/kmicms/pages/infra/templates/infra/netbox_device_index.html new file mode 100644 index 0000000..4048a78 --- /dev/null +++ b/kmicms/pages/infra/templates/infra/netbox_device_index.html @@ -0,0 +1,38 @@ +{% extends "layouts/page.html" %} +{% load wagtailcore_tags wagtailroutablepage_tags %} + +{% block title %}Devices{% endblock %} + +{% block content %} +
+

Devices

+ {% include "infra/inc/breadcrumbs_subpage.html" with title="Devices" %} + +

{{ page.device_description|richtext}}

+ + + + + + + + + + + + + + {% for device in devices %} + + + + + + + + + {% endfor %} + +
HostnameStatusRolePlatformLocation
{{ device.name }}{{ device.status }}{% include 'infra/inc/colored_badge.html' with info=device.role %}{% if device.platform %}{{ device.platform.name }}{% else %}-{% endif %}{{ device.rack.name }}View
+
+{% endblock %} \ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/netbox_device_view.html b/kmicms/pages/infra/templates/infra/netbox_device_view.html new file mode 100644 index 0000000..0356126 --- /dev/null +++ b/kmicms/pages/infra/templates/infra/netbox_device_view.html @@ -0,0 +1,32 @@ +{% extends "layouts/page.html" %} +{% load wagtailroutablepage_tags %} + +{% block title %}Device: {{ device.name }}{% endblock %} + +{% block content %} +
+

+ Device: {{ device.name }} +

+ {% routablepageurl page 'device_index' as device_index_url %} + {% include "infra/inc/breadcrumbs_subpage.html" with title=device.name parent_title="Devices" parent_link=device_index_url %} + +
+
+ {% include "infra/inc/server_dl.html" with server=device %} +
+
+
+
Rack
+
{{ device.rack.name }}
+
Location
+
{{ device.location.name }}
+
Type
+
{{ device.device_type.manufacturer.name }} {{ device.device_type.model }}
+
+
+
+ + {% include "infra/inc/interface_table.html" with interfaces=device.interfaces %} +
+{% endblock %} \ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/netbox_error.html b/kmicms/pages/infra/templates/infra/netbox_error.html new file mode 100644 index 0000000..57bafc8 --- /dev/null +++ b/kmicms/pages/infra/templates/infra/netbox_error.html @@ -0,0 +1,14 @@ +{% extends "layouts/page.html" %} +{% load wagtailroutablepage_tags %} + +{% block title %}{{ page.title}}: Unable to load{% endblock %} + +{% block content %} +
+

{{ page.title}}: Unable to load

+ {% include "components/breadcrumbs.html" %} + +

The page is unable to load as our IPAM database is currently not responding.

+

Please try reloading the page or try again later.

+
+{% endblock %} \ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/netbox_infrastructure_page.html b/kmicms/pages/infra/templates/infra/netbox_infrastructure_page.html new file mode 100644 index 0000000..494bffe --- /dev/null +++ b/kmicms/pages/infra/templates/infra/netbox_infrastructure_page.html @@ -0,0 +1,28 @@ +{% extends "layouts/page.html" %} + +{% load wagtailcore_tags wagtailroutablepage_tags %} + +{% block content %} +
+

{{ page.title }}

+ {% include "components/breadcrumbs.html" %} + +

{{ page.introduction|richtext}}

+ +
+
+
Devices
+

{{ page.device_description|richtext}}

+ View devices +
+
+ +
+
+
Virtual Machines
+

{{ page.vm_description|richtext}}

+ View VMs +
+
+
+{% endblock %} \ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/netbox_vm_index.html b/kmicms/pages/infra/templates/infra/netbox_vm_index.html new file mode 100644 index 0000000..3d15418 --- /dev/null +++ b/kmicms/pages/infra/templates/infra/netbox_vm_index.html @@ -0,0 +1,38 @@ +{% extends "layouts/page.html" %} +{% load wagtailcore_tags wagtailroutablepage_tags %} + +{% block title %}Virtual Machines{% endblock %} + +{% block content %} +
+

Virtual Machines

+ {% include "infra/inc/breadcrumbs_subpage.html" with title="Virtual Machines" %} + +

{{ page.vm_description|richtext}}

+ + + + + + + + + + + + + + {% for vm in vms %} + + + + + + + + + {% endfor %} + +
HostnameStatusRolePlatformCluster
{{ vm.name }}{{ vm.status }}{% include 'infra/inc/colored_badge.html' with info=vm.role %}{% if vm.platform %}{{ vm.platform.name }}{% else %}-{% endif %}{{ vm.cluster.name }}View
+
+{% endblock %} \ No newline at end of file diff --git a/kmicms/pages/infra/templates/infra/netbox_vm_view.html b/kmicms/pages/infra/templates/infra/netbox_vm_view.html new file mode 100644 index 0000000..f3daac6 --- /dev/null +++ b/kmicms/pages/infra/templates/infra/netbox_vm_view.html @@ -0,0 +1,34 @@ +{% extends "layouts/page.html" %} +{% load wagtailroutablepage_tags %} + +{% block title %}VM: {{ vm.name }}{% endblock %} + +{% block content %} +
+

+ VM: {{ vm.name }} +

+ {% routablepageurl page 'vm_index' as vm_index_url %} + {% include "infra/inc/breadcrumbs_subpage.html" with title=vm.name parent_title="Virtual Machines" parent_link=vm_index_url %} + +
+
+ {% include "infra/inc/server_dl.html" with server=vm %} +
+
+
+
vCPUs
+
{{ vm.vcpus }}
+
Memory
+
{{ vm.memory }}MB
+
Storage
+
{{ vm.disk }}GB
+
Cluster / Host
+
{{ vm.cluster.name }}
+
+
+
+ + {% include "infra/inc/interface_table.html" with interfaces=vm.interfaces %} +
+{% endblock %} \ No newline at end of file diff --git a/kmicms/pages/standard_page/models.py b/kmicms/pages/standard_page/models.py index 07acec8..2582cdf 100644 --- a/kmicms/pages/standard_page/models.py +++ b/kmicms/pages/standard_page/models.py @@ -13,7 +13,7 @@ class StandardPage(Page): show_title = models.BooleanField(help_text="Show page title at top of page?", default=True) parent_page_types = ["home.HomePage", "standard_page.StandardPage"] - subpage_types = ["standard_page.StandardPage", "contact.ContactFormPage"] + subpage_types = ["standard_page.StandardPage", "contact.ContactFormPage", "infra.NetboxInfrastructurePage"] content_panels = [ TitleFieldPanel("title"), diff --git a/kmicms/templates/components/breadcrumbs.html b/kmicms/templates/components/breadcrumbs.html new file mode 100644 index 0000000..6d1aef1 --- /dev/null +++ b/kmicms/templates/components/breadcrumbs.html @@ -0,0 +1,20 @@ +{% load wagtailcore_tags %} + +{% block breadcrumbs %} + {% if page.get_ancestors|length > 1 %} + + {% endif %} +{% endblock breadcrumbs %} \ No newline at end of file From 0569c2db95a23f63d34ccf25f443616e3909921d Mon Sep 17 00:00:00 2001 From: Dan Trickey Date: Thu, 28 Dec 2023 17:49:51 +0000 Subject: [PATCH 2/6] Add annotations future to graphql.py --- kmicms/integrations/netbox/graphql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kmicms/integrations/netbox/graphql.py b/kmicms/integrations/netbox/graphql.py index ba26809..8483a1c 100644 --- a/kmicms/integrations/netbox/graphql.py +++ b/kmicms/integrations/netbox/graphql.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from typing import Any From 5411cc34bf7e4667962d383d9d41c6ce3b3ee221 Mon Sep 17 00:00:00 2001 From: Dan Trickey Date: Thu, 28 Dec 2023 18:05:33 +0000 Subject: [PATCH 3/6] Implement dns_name filter tag Formats with a sown.org.uk domain if not fully qualified --- .github/workflows/test.yml | 4 +++- Makefile | 12 +++++++++--- kmicms/pages/infra/templatetags/__init__.py | 0 kmicms/pages/infra/templatetags/infra_tags.py | 13 +++++++++++++ kmicms/pages/infra/tests/__init__.py | 0 kmicms/pages/infra/tests/test_templatetags.py | 12 ++++++++++++ 6 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 kmicms/pages/infra/templatetags/__init__.py create mode 100644 kmicms/pages/infra/templatetags/infra_tags.py create mode 100644 kmicms/pages/infra/tests/__init__.py create mode 100644 kmicms/pages/infra/tests/test_templatetags.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3df8472..76aed2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,4 +36,6 @@ jobs: - name: Lint run: make lint - name: Django Static Checks - run: make check \ No newline at end of file + run: make check + - name: Unit tests + run: make test \ No newline at end of file diff --git a/Makefile b/Makefile index 8602e63..b2235b4 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -.PHONY: all check clean dev format format-check lint lint-fix +.PHONY: all check clean dev format format-check lint lint-fix test test-cov PYMODULE:=kmicms MANAGEPY:=$(CMD) ./$(PYMODULE)/manage.py APPS:=kmicms accounts core integrations pages -all: format lint check +all: format lint check test format: ruff format $(PYMODULE) @@ -27,4 +27,10 @@ dev: $(MANAGEPY) runserver clean: - git clean -Xdf # Delete all files in .gitignore \ No newline at end of file + git clean -Xdf # Delete all files in .gitignore + +test: | $(PYMODULE) + cd kmicms && DJANGO_SETTINGS_MODULE=kmicms.settings pytest --cov=. $(APPS) $(PYMODULE) + +test-cov: + cd kmicms && DJANGO_SETTINGS_MODULE=kmicms.settings pytest --cov=. $(APPS) $(PYMODULE) --cov-report html \ No newline at end of file diff --git a/kmicms/pages/infra/templatetags/__init__.py b/kmicms/pages/infra/templatetags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kmicms/pages/infra/templatetags/infra_tags.py b/kmicms/pages/infra/templatetags/infra_tags.py new file mode 100644 index 0000000..7bd7f75 --- /dev/null +++ b/kmicms/pages/infra/templatetags/infra_tags.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from django import template + +register = template.Library() + + +@register.filter(name="format_dns") +def format_dns(value: str, default_domain: str = "sown.org.uk") -> str: + if "." in value: + return value + else: + return f"{value}.{default_domain}" diff --git a/kmicms/pages/infra/tests/__init__.py b/kmicms/pages/infra/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kmicms/pages/infra/tests/test_templatetags.py b/kmicms/pages/infra/tests/test_templatetags.py new file mode 100644 index 0000000..2c66389 --- /dev/null +++ b/kmicms/pages/infra/tests/test_templatetags.py @@ -0,0 +1,12 @@ +from pages.infra.templatetags.infra_tags import format_dns + + +class TestFormatDNSTemplateFilter: + def test_fqdn(self) -> None: + assert format_dns("www.google.com") == "www.google.com" + + def test_not_fully_qualified(self) -> None: + assert format_dns("gw-b100") == "gw-b100.sown.org.uk" + + def test_not_fully_qualified_custom_domain(self) -> None: + assert format_dns("gw-b100", "suws.org.uk") == "gw-b100.suws.org.uk" From 7bfab080e1518fbdb3b90ebf673934e03f52bf4b Mon Sep 17 00:00:00 2001 From: Dan Trickey Date: Thu, 28 Dec 2023 18:36:30 +0000 Subject: [PATCH 4/6] Improve interface table template --- .../templates/infra/inc/interface_table.html | 17 ++++++++++++----- .../infra/templates/infra/inc/server_dl.html | 16 +++++++++------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/kmicms/pages/infra/templates/infra/inc/interface_table.html b/kmicms/pages/infra/templates/infra/inc/interface_table.html index 3e586d3..f7a48c1 100644 --- a/kmicms/pages/infra/templates/infra/inc/interface_table.html +++ b/kmicms/pages/infra/templates/infra/inc/interface_table.html @@ -1,3 +1,4 @@ +{% load infra_tags %} {% if interfaces %}

Interfaces

@@ -16,11 +17,17 @@

Interfaces

{% endfor %} diff --git a/kmicms/pages/infra/templates/infra/inc/server_dl.html b/kmicms/pages/infra/templates/infra/inc/server_dl.html index b96c14e..db9b575 100644 --- a/kmicms/pages/infra/templates/infra/inc/server_dl.html +++ b/kmicms/pages/infra/templates/infra/inc/server_dl.html @@ -6,14 +6,16 @@
{% include 'infra/inc/colored_badge.html' with info=server.role %}
Platform
-
{{ server.platform.name }}
+
{% if server.platform.name%}{{ server.platform.name }}{% else %}-{% endif %}
-
Tags
-
- {% for tag in server.tags %} - {% include "infra/inc/colored_badge.html" with info=tag %} - {% endfor %} -
+ {% if server.tags %} +
Tags
+
+ {% for tag in server.tags %} + {% include "infra/inc/colored_badge.html" with info=tag %} + {% endfor %} +
+ {% endif %} {% if server.description %}
Description
From 310fe096c2abb0019c9b52a51d026bff9b5e83c3 Mon Sep 17 00:00:00 2001 From: Dan Trickey Date: Thu, 28 Dec 2023 18:48:27 +0000 Subject: [PATCH 5/6] Add responses to dev requirements --- requirements-dev.in | 1 + requirements-dev.txt | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/requirements-dev.in b/requirements-dev.in index a4f0851..241d1f5 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -9,3 +9,4 @@ ruff pytest pytest-cov pytest-django +responses \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 4f6813b..bdf5f7d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -201,6 +201,8 @@ pytz==2023.3.post1 # django-modelcluster # djangorestframework # l18n +pyyaml==6.0.1 + # via responses rcssmin==1.1.1 # via # -r requirements.txt @@ -210,7 +212,10 @@ redis==5.0.1 requests==2.31.0 # via # -r requirements.txt + # responses # wagtail +responses==0.24.1 + # via -r requirements-dev.in rjsmin==1.2.1 # via # -r requirements.txt @@ -246,6 +251,7 @@ urllib3==2.0.7 # via # -r requirements.txt # requests + # responses wagtail==5.2.2 # via -r requirements.txt webencodings==0.5.1 From 27adf1ff1e04f56a983bba3c862cc45d3d8bedd4 Mon Sep 17 00:00:00 2001 From: Dan Trickey Date: Thu, 28 Dec 2023 19:11:31 +0000 Subject: [PATCH 6/6] Add primitive tests for the GraphQL client --- kmicms/integrations/netbox/client.py | 83 ++++++++++++++++++++++-- kmicms/integrations/netbox/graphql.py | 64 ------------------ kmicms/integrations/tests/__init__.py | 0 kmicms/integrations/tests/conftest.py | 10 +++ kmicms/integrations/tests/test_netbox.py | 60 +++++++++++++++++ kmicms/pages/infra/models.py | 8 ++- 6 files changed, 153 insertions(+), 72 deletions(-) delete mode 100644 kmicms/integrations/netbox/graphql.py create mode 100644 kmicms/integrations/tests/__init__.py create mode 100644 kmicms/integrations/tests/conftest.py create mode 100644 kmicms/integrations/tests/test_netbox.py diff --git a/kmicms/integrations/netbox/client.py b/kmicms/integrations/netbox/client.py index c23895b..00fbffa 100644 --- a/kmicms/integrations/netbox/client.py +++ b/kmicms/integrations/netbox/client.py @@ -1,15 +1,84 @@ +from __future__ import annotations + from typing import Any -from django.conf import settings +import requests from django.core.cache import cache -from .graphql import netbox_query +from .exceptions import NetboxRequestError from .queries import GET_DEVICE_QUERY, GET_VM_QUERY, LIST_DEVICE_QUERY, LIST_VM_QUERY class NetboxClient: - def __init__(self, *, cache_ttl_seconds: int = settings.NETBOX_CACHE_TTL) -> None: + def __init__( + self, *, graphql_endpoint: str, api_token: str, cache_ttl_seconds: int, request_timeout: float + ) -> None: + self.graphql_endpoint = graphql_endpoint + self.api_token = api_token self.cache_ttl_seconds = cache_ttl_seconds + self.request_timeout = request_timeout + + def _get_headers(self) -> dict[str, str]: + return { + "Authorization": f"Token {self.api_token}", + "Accept": "application/json", + "Content-Type": "application/json", + } + + def _get_payload(self, query: str, variables: dict[str, str] | None = None) -> dict[str, str]: + payload = { + "query": query, + } + + if variables is not None: + payload["variables"] = variables + + return payload + + def _query(self, query: str, variables: dict[str, str] | None = None) -> dict[str, Any]: # noqa: FA102 + payload = self._get_payload(query, variables) + + try: + resp = requests.post( + self.graphql_endpoint, + headers=self._get_headers(), + json=payload, + timeout=self.request_timeout, + ) + except ConnectionError as e: + raise NetboxRequestError("Unable to connect to netbox") from e + except requests.RequestException as e: + raise NetboxRequestError("Error when requesting data from netbox") from e + + try: + resp.raise_for_status() + except requests.RequestException as e: + # Include GraphQL errors in the exception message. + try: + data = resp.json() + except requests.exceptions.JSONDecodeError: + message = str(e) + else: + errors = data.get("errors") + message = f"{e}: GraphQL errors: {errors}" + raise NetboxRequestError(message) from e + + try: + gql_response = resp.json() + except requests.exceptions.JSONDecodeError as e: + raise NetboxRequestError("Netbox returned invalid JSON") from e + + # Check for and raise any GraphQL errors from successful responses. + if "errors" in gql_response: + errors = gql_response["errors"] + raise NetboxRequestError(f"Invalid GraphQL response: {errors}") + + try: + return gql_response["data"] + except KeyError as e: + raise NetboxRequestError( + "Netbox API response did not contain data.", + ) from e def get_device(self, device_id: int) -> dict[str, Any]: cache_key = f"netbox__device__{device_id}" @@ -17,7 +86,7 @@ def get_device(self, device_id: int) -> dict[str, Any]: if cache_value := cache.get(cache_key): return cache_value - device = netbox_query(GET_DEVICE_QUERY, {"deviceId": device_id})["device"] + device = self._query(GET_DEVICE_QUERY, {"deviceId": device_id})["device"] cache.set(cache_key, device, timeout=self.cache_ttl_seconds) return device @@ -28,7 +97,7 @@ def get_vm(self, vm_id: int) -> dict[str, Any]: if cache_value := cache.get(cache_key): return cache_value - vm = netbox_query(GET_VM_QUERY, {"VMId": vm_id})["virtual_machine"] + vm = self._query(GET_VM_QUERY, {"VMId": vm_id})["virtual_machine"] cache.set(cache_key, vm, timeout=self.cache_ttl_seconds) return vm @@ -41,7 +110,7 @@ def list_devices( if cache_value := cache.get(cache_key): return cache_value - device = netbox_query(LIST_DEVICE_QUERY)["device_list"] + device = self._query(LIST_DEVICE_QUERY)["device_list"] cache.set(cache_key, device, timeout=self.cache_ttl_seconds) return device @@ -54,7 +123,7 @@ def list_vms( if cache_value := cache.get(cache_key): return cache_value - vms = netbox_query(LIST_VM_QUERY)["virtual_machine_list"] + vms = self._query(LIST_VM_QUERY)["virtual_machine_list"] cache.set(cache_key, vms, timeout=self.cache_ttl_seconds) return vms diff --git a/kmicms/integrations/netbox/graphql.py b/kmicms/integrations/netbox/graphql.py deleted file mode 100644 index 8483a1c..0000000 --- a/kmicms/integrations/netbox/graphql.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any - -import requests -from django.conf import settings - -from .exceptions import NetboxRequestError - - -def netbox_query(query: str, variables: dict[str, str] | None = None) -> dict[str, Any]: # noqa: FA102 - payload = { - "query": query, - } - - if variables is not None: - payload["variables"] = variables - - try: - resp = requests.post( - settings.NETBOX_GRAPHQL_ENDPOINT, - headers={ - "Authorization": f"Token {settings.NETBOX_API_TOKEN}", - "Accept": "application/json", - "Content-Type": "application/json", - }, - json=payload, - timeout=settings.NETBOX_REQUEST_TIMEOUT, - ) - except ConnectionError as e: - raise NetboxRequestError("Unable to connect to netbox") from e - except requests.RequestException as e: - raise NetboxRequestError("Error when requesting data from netbox") from e - - try: - resp.raise_for_status() - except requests.RequestException as e: - # Include GraphQL errors in the exception message. - try: - data = resp.json() - except json.JSONDecodeError: - message = str(e) - else: - errors = data.get("errors") - message = f"{e}: GraphQL errors: {errors}" - raise NetboxRequestError(message) from e - - try: - gql_response = resp.json() - except json.JSONDecodeError as e: - raise NetboxRequestError("Netbox returned invalid JSON") from e - - # Check for and raise any GraphQL errors from successful responses. - if "errors" in gql_response: - errors = gql_response["errors"] - raise NetboxRequestError(f"Invalid GraphQL response: {errors}") - - try: - return gql_response["data"] - except KeyError as e: - raise NetboxRequestError( - "Netbox API response did not contain data.", - ) from e diff --git a/kmicms/integrations/tests/__init__.py b/kmicms/integrations/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kmicms/integrations/tests/conftest.py b/kmicms/integrations/tests/conftest.py new file mode 100644 index 0000000..d6996fc --- /dev/null +++ b/kmicms/integrations/tests/conftest.py @@ -0,0 +1,10 @@ +from collections.abc import Iterator + +import pytest +import responses + + +@pytest.fixture +def mocked_responses() -> Iterator[responses.RequestsMock]: + with responses.RequestsMock() as requests_mock: + yield requests_mock diff --git a/kmicms/integrations/tests/test_netbox.py b/kmicms/integrations/tests/test_netbox.py new file mode 100644 index 0000000..e5f8409 --- /dev/null +++ b/kmicms/integrations/tests/test_netbox.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from http import HTTPStatus +from typing import Any + +import pytest +from responses import RequestsMock, matchers + +from integrations.netbox import NetboxClient + + +class TestNetboxClient: + @pytest.fixture + def client(self) -> NetboxClient: + return NetboxClient( + graphql_endpoint="https://netbox.invalid/graphql/", + api_token="abc", # noqa: S106 + cache_ttl_seconds=300, + request_timeout=0.5, + ) + + def test_get_headers(self, client: NetboxClient) -> None: + expected_headers = { + "Authorization": "Token abc", + "Accept": "application/json", + "Content-Type": "application/json", + } + assert expected_headers == client._get_headers() + + @pytest.mark.parametrize( + ("query", "variables", "expected_payload"), + [ + pytest.param("query", None, {"query": "query"}, id="no-variables"), + pytest.param("query", {}, {"query": "query", "variables": {}}, id="empty-variables"), + pytest.param("query", {"a": 1}, {"query": "query", "variables": {"a": 1}}, id="variables"), + ], + ) + def test_get_payload( + self, + client: NetboxClient, + query: str, + variables: dict[str, str] | None, + expected_payload: dict[str, Any], + ) -> None: + assert client._get_payload(query, variables) == expected_payload + + def test_query(self, client: NetboxClient, mocked_responses: RequestsMock) -> None: + mocked_responses.post( + "https://netbox.invalid/graphql/", + match=[ + matchers.header_matcher( + {"Content-Type": "application/json", "Accept": "application/json", "Authorization": "Token abc"}, + ), + ], + json={"data": {"a": "b"}}, + status=HTTPStatus.OK, + content_type="application/json", + ) + data = client._query("query") + assert data == {"a": "b"} diff --git a/kmicms/pages/infra/models.py b/kmicms/pages/infra/models.py index 8ebc44b..c749d2f 100644 --- a/kmicms/pages/infra/models.py +++ b/kmicms/pages/infra/models.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.http import Http404, HttpRequest, HttpResponse from wagtail.admin.panels import FieldPanel, TitleFieldPanel from wagtail.contrib.routable_page.models import RoutablePageMixin, path @@ -23,7 +24,12 @@ class NetboxInfrastructurePage(RoutablePageMixin, Page): ] def _get_client(self) -> NetboxClient: - return NetboxClient() + return NetboxClient( + graphql_endpoint=settings.NETBOX_GRAPHQL_ENDPOINT, + api_token=settings.NETBOX_API_TOKEN, + cache_ttl_seconds=settings.NETBOX_CACHE_TTL, + request_timeout=settings.NETBOX_REQUEST_TIMEOUT, + ) def _handle_error(self, request: HttpRequest) -> HttpResponse: return self.render(
{% if interface.enabled %}Enabled{% else %}Disabled{% endif %} {% if interface.description %}{{ interface.description}}{% else %}-{% endif %} -
    - {% for address in interface.ip_addresses %} -
  • {{ address.display }}: {{ address.dns_name }}
    • - {% endfor %} -
+ {% if interface.ip_addresses %} +
    + {% for address in interface.ip_addresses %} +
  • + {{ address.display }}: {{ address.dns_name|format_dns }} +
    • + {% endfor %} +
+ {% else %} + - + {% endif %}