Flexible mapping provider to parser

development/creds.example.env

@@ -1,5 +1,5 @@
 POSTGRES_PASSWORD=notverysecurepwd
-REDIS_PASSWORD=notverysecurepwd
+NAUTOBOT_REDIS_PASSWORD=notverysecurepwd
 SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
 NAUTOBOT_CREATE_SUPERUSER=true
 NAUTOBOT_SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567

development/dev.env

@@ -9,12 +9,13 @@ NAUTOBOT_ROOT=/opt/nautobot
 POSTGRES_DB=nautobot
 POSTGRES_HOST=postgres
 POSTGRES_USER=nautbot
-REDIS_HOST=redis
-REDIS_PORT=6379
+NAUTOBOT_REDIS_HOST=redis
+NAUTOBOT_REDIS_PORT=6379
 # REDIS_SSL=True
 # Uncomment REDIS_SSL if using SSL
 SUPERUSER_EMAIL=admin@example.com
 SUPERUSER_NAME=admin
+NAUTOBOT_LOG_LEVEL=DEBUG
 
 # Enable Oauth workflow for development, http
 OAUTHLIB_INSECURE_TRANSPORT="1"

development/docker-compose.base.yml

@@ -25,8 +25,29 @@ services:
     <<: *nautobot-base
   worker:
     entrypoint: "nautobot-server rqworker"
+    healthcheck:
+      interval: 5s
+      timeout: 5s
+      start_period: 5s
+      retries: 3
+      test: ["CMD", "nautobot-server", "health_check"]
     depends_on:
       - "nautobot"
     healthcheck:
       disable: true
     <<: *nautobot-base
+  celery_worker:
+    healthcheck:
+      interval: 5s
+      timeout: 5s
+      start_period: 5s
+      retries: 3
+      test: ["CMD", "nautobot-server", "health_check"]
+    depends_on:
+      - nautobot
+      - redis
+    entrypoint:
+      - "sh"
+      - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env
+      - "nautobot-server celery worker -B -l $$NAUTOBOT_LOG_LEVEL" ## $$ because of docker-compose
+    <<: *nautobot-base

development/docker-compose.dev.yml

@@ -3,14 +3,19 @@
 # any override will need to include these volumes to use them.
 # see:  https://github.com/docker/compose/issues/3729
 ---
+x-nautobot-base: &nautobot-base
+  volumes:
+    - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
+    - "../:/source"
+
 version: "3.4"
 services:
   nautobot:
+    healthcheck:
+      disable: true
     command: "nautobot-server runserver 0.0.0.0:8080"
-    volumes:
-      - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
-      - "../:/source"
+    <<: *nautobot-base
+  celery_worker:
+    <<: *nautobot-base
   worker:
-    volumes:
-      - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
-      - "../:/source"
+    <<: *nautobot-base

development/docker-compose.requirements.yml

@@ -8,18 +8,18 @@ services:
       - "creds.env"
     volumes:
       - "postgres_data:/var/lib/postgresql/data"
-    ports:
-      - "5432:5432"
+    # ports:
+    #   - "5432:5432"
   redis:
     image: "redis:6-alpine"
     command:
       - "sh"
       - "-c" # this is to evaluate the $REDIS_PASSWORD from the env
-      - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD"
+      - "redis-server --appendonly yes --requirepass $$NAUTOBOT_REDIS_PASSWORD"
     env_file:
       - "dev.env"
       - "creds.env"
-    ports:
-      - "6379:6379"
+    # ports:
+    #   - "6379:6379"
 volumes:
   postgres_data: {}

development/nautobot_config.py

@@ -1,15 +1,12 @@
-#########################
-#                       #
-#   Required settings   #
-#                       #
-#########################
+"""Nautobot development configuration file."""
 
 import os
 import sys
 
-from distutils.util import strtobool
 from django.core.exceptions import ImproperlyConfigured
-from nautobot.core import settings
+from nautobot.core.settings import *  # noqa: F403
+from nautobot.core.settings_funcs import is_truthy, parse_redis_connection
+
 
 # Enforce required configuration parameters
 for key in [
@@ -18,29 +15,12 @@
     "POSTGRES_USER",
     "POSTGRES_HOST",
     "POSTGRES_PASSWORD",
-    "REDIS_HOST",
-    "REDIS_PASSWORD",
     "SECRET_KEY",
 ]:
     if not os.environ.get(key):
         raise ImproperlyConfigured(f"Required environment variable {key} is missing.")
 
 
-def is_truthy(arg):
-    """Convert "truthy" strings into Booleans.
-
-    Examples:
-        >>> is_truthy('yes')
-        True
-    Args:
-        arg (str): Truthy string (True values are y, yes, t, true, on and 1; false values are n, no,
-        f, false, off and 0. Raises ValueError if val is anything else.
-    """
-    if isinstance(arg, bool):
-        return arg
-    return bool(strtobool(arg))
-
-
 TESTING = len(sys.argv) > 1 and sys.argv[1] == "test"
 
 # This is a list of valid fully-qualified domain names (FQDNs) for the Nautobot server. Nautobot will not permit write
@@ -63,62 +43,22 @@ def is_truthy(arg):
     }
 }
 
-# Nautobot uses RQ for task scheduling. These are the following defaults.
-# For detailed configuration see: https://github.com/rq/django-rq#installation
-RQ_QUEUES = {
-    "default": {
-        "HOST": os.getenv("REDIS_HOST", "localhost"),
-        "PORT": os.getenv("REDIS_PORT", 6379),
-        "DB": 0,
-        "PASSWORD": os.getenv("REDIS_PASSWORD", ""),
-        "SSL": os.getenv("REDIS_SSL", False),
-        "DEFAULT_TIMEOUT": 300,
-    },
-    "webhooks": {
-        "HOST": os.getenv("REDIS_HOST", "localhost"),
-        "PORT": os.getenv("REDIS_PORT", 6379),
-        "DB": 0,
-        "PASSWORD": os.getenv("REDIS_PASSWORD", ""),
-        "SSL": os.getenv("REDIS_SSL", False),
-        "DEFAULT_TIMEOUT": 300,
-    },
-    "custom_fields": {
-        "HOST": os.getenv("REDIS_HOST", "localhost"),
-        "PORT": os.getenv("REDIS_PORT", 6379),
-        "DB": 0,
-        "PASSWORD": os.getenv("REDIS_PASSWORD", ""),
-        "SSL": os.getenv("REDIS_SSL", False),
-        "DEFAULT_TIMEOUT": 300,
-    },
-    # "with-sentinel": {
-    #     "SENTINELS": [
-    #         ("mysentinel.redis.example.com", 6379)
-    #         ("othersentinel.redis.example.com", 6379)
-    #     ],
-    #     "MASTER_NAME": 'nautobot",
-    #     "DB": 0,
-    #     "PASSWORD": "",
-    #     "SOCKET_TIMEOUT": None,
-    #     'CONNECTION_KWARGS': {
-    #         'socket_connect_timeout': 10,
-    #     },
-    # },
-    "check_releases": {
-        "HOST": os.getenv("REDIS_HOST", "localhost"),
-        "PORT": os.getenv("REDIS_PORT", 6379),
-        "DB": 0,
-        "PASSWORD": os.getenv("REDIS_PASSWORD", ""),
-        "SSL": os.getenv("REDIS_SSL", False),
-        "DEFAULT_TIMEOUT": 300,
-    },
-}
 
 # Nautobot uses Cacheops for database query caching. These are the following defaults.
 # For detailed configuration see: https://github.com/Suor/django-cacheops#setup
-REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
-REDIS_PORT = os.getenv("REDIS_PORT", 6379)
-REDIS_PASS = os.getenv("REDIS_PASSWORD", "")
-CACHEOPS_REDIS = f"redis://:{REDIS_PASS}@{REDIS_HOST}:{REDIS_PORT}/1"
+
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": parse_redis_connection(redis_database=0),
+        "TIMEOUT": 300,
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    }
+}
+CACHEOPS_REDIS = parse_redis_connection(redis_database=1)
+
 
 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file.
 # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and
@@ -383,7 +323,7 @@ def is_truthy(arg):
 # Django Debug Toolbar
 DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: DEBUG and not TESTING}
 
-if "debug_toolbar" not in EXTRA_INSTALLED_APPS:
-    EXTRA_INSTALLED_APPS.append("debug_toolbar")
-if "debug_toolbar.middleware.DebugToolbarMiddleware" not in settings.MIDDLEWARE:
-    settings.MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware")
+if "debug_toolbar" not in INSTALLED_APPS:  # noqa: F405
+    INSTALLED_APPS.append("debug_toolbar")  # noqa: F405
+if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE:  # noqa: F405
+    MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware")  # noqa: F405

nautobot_circuit_maintenance/__init__.py

@@ -6,8 +6,8 @@
 from nautobot.extras.plugins import PluginConfig
 
 
-def custom_field_extension(sender, **kwargs):  # pylint: disable=unused-argument
-    """Add extended custom field."""
+def custom_fields_extension(sender, **kwargs):  # pylint: disable=unused-argument
+    """Add extended custom fields."""
     # pylint: disable=import-outside-toplevel
     from django.contrib.contenttypes.models import ContentType
     from nautobot.circuits.models import Provider
@@ -18,7 +18,12 @@ def custom_field_extension(sender, **kwargs):  # pylint: disable=unused-argument
         {
             "name": "emails_circuit_maintenances",
             "type": CustomFieldTypeChoices.TYPE_TEXT,
-            "label": "Emails for Circuit Maintenance plugin",
+            "label": "Emails for Circuit Maintenance plugin.",
+        },
+        {
+            "name": "provider_parser_circuit_maintenances",
+            "type": CustomFieldTypeChoices.TYPE_TEXT,
+            "label": "Provider Parser for Circuit Maintenance plugin.",
         },
     ]:
         field, _ = CustomField.objects.get_or_create(name=provider_cf_dict["name"], defaults=provider_cf_dict)
@@ -66,7 +71,7 @@ class CircuitMaintenanceConfig(PluginConfig):
 
     def ready(self):
         super().ready()
-        post_migrate.connect(custom_field_extension, sender=self)
+        post_migrate.connect(custom_fields_extension, sender=self)
         post_migrate.connect(import_notification_sources, sender=self)
 
 

nautobot_circuit_maintenance/custom_validators.py

@@ -1,6 +1,9 @@
 """Custom Validators definition."""
 from nautobot.circuits.models import Provider
 from nautobot.extras.plugins import PluginCustomValidator
+from nautobot.extras.models import CustomField
+
+from circuit_maintenance_parser import SUPPORTED_PROVIDER_NAMES
 
 
 class ProviderEmailValidator(PluginCustomValidator):
@@ -26,4 +29,28 @@ def clean(self):
                         )
 
 
-custom_validators = [ProviderEmailValidator]
+class ProviderParserValidator(PluginCustomValidator):
+    """Custom validator to validate that Provider's parser exists in the Parser library."""
+
+    model = "circuits.provider"
+
+    def clean(self):
+        """Validate that the Provider's parser exists in the Parser library."""
+        provider_mapping = (
+            self.context["object"]
+            .get_custom_fields()
+            .get(CustomField.objects.get(name="provider_parser_circuit_maintenances"))
+        )
+
+        if provider_mapping and provider_mapping.lower() not in SUPPORTED_PROVIDER_NAMES:
+            self.validation_error(
+                {
+                    "cf_provider_parser_circuit_maintenances": (
+                        f"{provider_mapping} is not one of the supported Providers in the "
+                        f"circuit-maintenance-parser library: {SUPPORTED_PROVIDER_NAMES}."
+                    )
+                }
+            )
+
+
+custom_validators = [ProviderEmailValidator, ProviderParserValidator]

nautobot_circuit_maintenance/handle_notifications/sources.py

@@ -266,12 +266,20 @@ def extract_provider_data_types(email_source: str) -> Tuple[str, str, str]:
             ]
             if email_source in sources:
                 provider_type = provider.slug
+                # If there is no custom provider mapping defined, we take the provider slug as default mapping
+                provider_parser_circuit_maintenances = provider.get_custom_fields().get(
+                    CustomField.objects.get(name="provider_parser_circuit_maintenances")
+                )
+                if provider_parser_circuit_maintenances:
+                    provider_mapping = provider_parser_circuit_maintenances.lower()
+                else:
+                    provider_mapping = provider_type
                 break
         else:
             return "", "", f"Sender email {email_source} is not registered for any circuit provider."
 
         try:
-            provider_data_types = get_provider_data_types(provider_type)
+            provider_data_types = get_provider_data_types(provider_mapping)
         except NonexistentParserError:
             return (
                 "",

nautobot_circuit_maintenance/migrations/0006_fake_migration_to_update_custom_fields.py

@@ -0,0 +1,13 @@
+# Generated manually just to trigger post_migration signal that is
+# used to create the Custom Fields
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("nautobot_circuit_maintenance", "0005_notificationsource__token"),
+    ]
+
+    operations = []

nautobot_circuit_maintenance/templates/nautobot_circuit_maintenance/parsednotification.html

@@ -12,12 +12,12 @@
   <div class="col-sm-8 col-md-9">
     <ol class="breadcrumb">
       <li><a href="{% url 'plugins:nautobot_circuit_maintenance:rawnotification_list' %}">ParsedNotification</a></li>
-      <li><a>{{ object.raw_notification.subject }}</a></li>
+      <li><a>{{ object }}</a></li>
     </ol>
   </div>
 </div>
 
-<h1>{{ object.raw_notification.subject }}</h1>
+<h1>{{ object }}</h1>
 <div class="pull-right noprint">{% custom_links object %}</div>
 
 {% endblock %}

nautobot_circuit_maintenance/tests/test_sources.py

@@ -141,17 +141,20 @@ def test_extract_provider_data_types_no_provider_parser(self):
 
     @parameterized.expand(
         [
-            ["ntt", {"text/calendar"}, False],
-            ["telstra", {"text/html", "text/calendar"}, False],
-            ["zayo", {"text/html"}, False],
-            ["unknown", "unknown", True],
+            ["ntt", {"text/calendar"}, "", False],
+            ["ntt", {"text/calendar"}, "eunetworks", False],
+            ["telstra", {"text/html", "text/calendar"}, "", False],
+            ["zayo", {"text/html"}, "", False],
+            ["unknown", "unknown", "", True],
         ]
     )
-    def test_extract_provider_data_types_ok(self, provider_type, data_types, error_message):
+    def test_extract_provider_data_types_ok(self, provider_type, data_types, provider_mapping, error_message):
         """Test for extract_provider_data_types."""
         email_source = "user@example.com"
         provider = Provider.objects.create(name=provider_type, slug=provider_type)
         provider.cf["emails_circuit_maintenances"] = email_source
+        if provider_mapping:
+            provider.cf["provider_parser_circuit_maintenances"] = provider_mapping
         provider.save()
 
         self.assertEqual(