Merge pull request #398 from nautobot/u/snaselj-napps-276-external-in…

…tegrations

External Integrations

changes/398.changed

@@ -0,0 +1 @@
+Changed Arista Cloud Vision jobs to optionally use ExternalIntegration.

development/run_example_job.py

@@ -0,0 +1,89 @@
+"""Executes a job locally for testing purposes.
+
+To run this script use the following command:
+
+```
+invoke nbshell \
+    --plain \
+    --file development/run_example_job.py \
+    --env RUN_SSOT_TARGET_JOB=False \
+    --env RUN_SSOT_JOB_DRY_RUN=True
+```
+
+Passing environment variables to the script is optional. The script will default to running the data source job with a dry run enabled.
+"""
+
+import json
+import os
+
+from django.core.management import call_command
+from nautobot.core.settings_funcs import is_truthy
+from nautobot.extras.choices import SecretsGroupAccessTypeChoices
+from nautobot.extras.choices import SecretsGroupSecretTypeChoices
+from nautobot.extras.models import ExternalIntegration
+from nautobot.extras.models import Job
+from nautobot.extras.models import Secret
+from nautobot.extras.models import SecretsGroup
+from nautobot.extras.models import SecretsGroupAssociation
+
+_TOKEN = 40 * "a"
+os.environ["NAUTOBOT_DEMO_TOKEN"] = _TOKEN
+
+_NAUTOBOT_DEMO_URL = "https://demo.nautobot.com"
+_DRY_RUN = is_truthy(os.getenv("RUN_SSOT_JOB_DRY_RUN", "True"))
+
+module_name = "nautobot_ssot.jobs.examples"
+is_target_job = is_truthy(os.getenv("RUN_SSOT_TARGET_JOB", "False"))
+job_class_name = "ExampleDataTarget" if is_target_job else "ExampleDataSource"
+
+job = Job.objects.get(module_name=module_name, job_class_name=job_class_name)
+if not job.enabled:
+    job.enabled = True
+    job.validated_save()
+
+nautobot_demo, created = ExternalIntegration.objects.get_or_create(
+    name="Nautobot Demo",
+    defaults={
+        "remote_url": _NAUTOBOT_DEMO_URL,
+        "verify_ssl": False,
+    },
+)
+
+if created:
+    secret = Secret.objects.create(
+        name="nautobot-demo-token",
+        provider="environment-variable",
+        parameters={"variable": "NAUTOBOT_DEMO_TOKEN"},
+    )
+    secrets_group = SecretsGroup.objects.create(name="Nautobot Demo Group")
+    SecretsGroupAssociation.objects.create(
+        secret=secret,
+        secrets_group=secrets_group,
+        access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP,
+        secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN,
+    )
+    nautobot_demo.secrets_group = secrets_group
+    nautobot_demo.validated_save()
+
+data: dict = {
+    "debug": True,
+    "dryrun": _DRY_RUN,
+    "memory_profiling": False,
+}
+
+if is_target_job:
+    data["target"] = str(nautobot_demo.pk)
+    data["target_url"] = _NAUTOBOT_DEMO_URL
+    data["target_token"] = _TOKEN
+else:
+    data["source"] = str(nautobot_demo.pk)
+    data["source_url"] = _NAUTOBOT_DEMO_URL
+    data["source_token"] = _TOKEN
+
+call_command(
+    "runjob",
+    f"{module_name}.{job_class_name}",
+    data=json.dumps(data),
+    username="admin",
+    local=True,  # Enable to run the job locally (not as a celery task)
+)

nautobot_ssot/__init__.py

@@ -74,6 +74,7 @@ class NautobotSSOTAppConfig(NautobotAppConfig):
         "aristacv_from_cloudvision_default_site": "",
         "aristacv_hostname_patterns": [],
         "aristacv_import_active": False,
+        "aristacv_external_integration_name": "",
         "aristacv_role_mappings": {},
         "aristacv_site_mappings": {},
         "aristacv_verify": True,

nautobot_ssot/integrations/aristacv/constant.py → nautobot_ssot/integrations/aristacv/constants.py

@@ -1,14 +1,18 @@
 """Storage of data that will not change throughout the life cycle of the application."""
 
-from django.conf import settings
-
-
-def _read_settings() -> dict:
-    config = settings.PLUGINS_CONFIG["nautobot_ssot"]
-    return config
-
-
-APP_SETTINGS = _read_settings()
+ARISTA_PLATFORM = "arista.eos.eos"
+CLOUDVISION_PLATFORM = "Arista EOS-CloudVision"
+DEFAULT_APPLY_IMPORT_TAG = False
+DEFAULT_CREATE_CONTROLLER = False
+DEFAULT_CVAAS_URL = "https://www.arista.io"
+DEFAULT_DELETE_DEVICES_ON_SYNC = False
+DEFAULT_DEVICE_ROLE = "network"
+DEFAULT_DEVICE_ROLE_COLOR = "ff0000"
+DEFAULT_DEVICE_STATUS = "cloudvision_imported"
+DEFAULT_DEVICE_STATUS_COLOR = "ff0000"
+DEFAULT_IMPORT_ACTIVE = False
+DEFAULT_SITE = "cloudvision_imported"
+DEFAULT_VERIFY_SSL = True
 
 PORT_TYPE_MAP = {
     "xcvr1000BaseT": "1000base-t",
@@ -83,7 +87,3 @@ def _read_settings() -> dict:
     "400GBASE-2FR4": "400gbase-x-osfp",
     "400GBASE-ZR": "400gbase-x-qsfpdd",
 }
-
-CLOUDVISION_PLATFORM = "Arista EOS-CloudVision"
-
-ARISTA_PLATFORM = "arista.eos.eos"

nautobot_ssot/integrations/aristacv/diffsync/adapters/cloudvision.py

@@ -7,7 +7,6 @@
 from diffsync import DiffSync
 from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound
 
-from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS
 from nautobot_ssot.integrations.aristacv.diffsync.models.cloudvision import (
     CloudvisionCustomField,
     CloudvisionDevice,
@@ -17,6 +16,7 @@
     CloudvisionIPAddress,
     CloudvisionIPAssignment,
 )
+from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig
 from nautobot_ssot.integrations.aristacv.utils import cloudvision
 
 
@@ -41,8 +41,13 @@ def __init__(self, *args, job=None, conn: cloudvision.CloudvisionApi, **kwargs):
 
     def load_devices(self):
         """Load devices from CloudVision."""
-        if APP_SETTINGS.get("aristacv_create_controller"):
-            cvp_version = cloudvision.get_cvp_version()
+        config: CloudVisionAppConfig = self.job.app_config
+        if config.hostname_patterns and not (config.site_mappings and config.role_mappings):
+            self.job.logger.warning(
+                "Configuration found for aristacv_hostname_patterns but no aristacv_site_mappings or aristacv_role_mappings. Please ensure your mappings are defined."
+            )
+        if config.create_controller:
+            cvp_version = cloudvision.get_cvp_version(config)
             cvp_ver_cf = self.cf(name="arista_eos", value=cvp_version, device_name="CloudVision")
             try:
                 self.add(cvp_ver_cf)
@@ -258,10 +263,4 @@ def load_device_tags(self, device):
 
     def load(self):
         """Load devices and associated data from CloudVision."""
-        if APP_SETTINGS.get("aristacv_hostname_patterns") and not (
-            APP_SETTINGS.get("aristacv_site_mappings") and APP_SETTINGS.get("aristacv_role_mappings")
-        ):
-            self.job.logger.warning(
-                "Configuration found for aristacv_hostname_patterns but no aristacv_site_mappings or aristacv_role_mappings. Please ensure your mappings are defined."
-            )
         self.load_devices()

nautobot_ssot/integrations/aristacv/diffsync/adapters/nautobot.py

@@ -11,7 +11,6 @@
 from diffsync import DiffSync
 from diffsync.exceptions import ObjectNotFound, ObjectAlreadyExists
 
-from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS
 from nautobot_ssot.integrations.aristacv.diffsync.models.nautobot import (
     NautobotDevice,
     NautobotCustomField,
@@ -21,6 +20,7 @@
     NautobotIPAssignment,
     NautobotPort,
 )
+from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig
 from nautobot_ssot.integrations.aristacv.utils import nautobot
 
 
@@ -166,8 +166,9 @@ def sync_complete(self, source: DiffSync, *args, **kwargs):
                     self.job.logger.warning(f"Deletion failed for protected object: {nautobot_object}. {err}")
             self.objects_to_delete[grouping] = []
 
+        config: CloudVisionAppConfig = self.job.app_config  # type: ignore
         # if Controller is created we need to ensure all imported Devices have RelationshipAssociation to it.
-        if APP_SETTINGS.get("aristacv_create_controller"):
+        if config.create_controller:
             self.job.logger.info("Creating Relationships between CloudVision and connected Devices.")
             controller_relation = OrmRelationship.objects.get(label="Controller -> Device")
             device_ct = ContentType.objects.get_for_model(OrmDevice)

nautobot_ssot/integrations/aristacv/diffsync/models/cloudvision.py

@@ -1,5 +1,4 @@
-"""Cloudvision DiffSync models for AristaCV SSoT."""
-from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS
+"""CloudVision DiffSync models for AristaCV SSoT."""
 from nautobot_ssot.integrations.aristacv.diffsync.models.base import (
     Device,
     CustomField,
@@ -9,11 +8,12 @@
     IPAssignment,
     Port,
 )
+from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig
 from nautobot_ssot.integrations.aristacv.utils.cloudvision import CloudvisionApi
 
 
 class CloudvisionDevice(Device):
-    """Cloudvision Device model."""
+    """CloudVision Device model."""
 
     @classmethod
     def create(cls, diffsync, ids, attrs):
@@ -30,7 +30,7 @@ def delete(self):
 
 
 class CloudvisionPort(Port):
-    """Cloudvision Port model."""
+    """CloudVision Port model."""
 
     @classmethod
     def create(cls, diffsync, ids, attrs):
@@ -47,7 +47,7 @@ def delete(self):
 
 
 class CloudvisionNamespace(Namespace):
-    """Cloudvision Namespace model."""
+    """CloudVision Namespace model."""
 
     @classmethod
     def create(cls, diffsync, ids, attrs):
@@ -67,7 +67,7 @@ def delete(self):
 
 
 class CloudvisionPrefix(Prefix):
-    """Cloudvision IPAdress model."""
+    """CloudVision IPAdress model."""
 
     @classmethod
     def create(cls, diffsync, ids, attrs):
@@ -87,7 +87,7 @@ def delete(self):
 
 
 class CloudvisionIPAddress(IPAddress):
-    """Cloudvision IPAdress model."""
+    """CloudVision IPAdress model."""
 
     @classmethod
     def create(cls, diffsync, ids, attrs):
@@ -107,7 +107,7 @@ def delete(self):
 
 
 class CloudvisionIPAssignment(IPAssignment):
-    """Cloudvision IPAssignment model."""
+    """CloudVision IPAssignment model."""
 
     @classmethod
     def create(cls, diffsync, ids, attrs):
@@ -127,24 +127,19 @@ def delete(self):
 
 
 class CloudvisionCustomField(CustomField):
-    """Cloudvision CustomField model."""
+    """CloudVision CustomField model."""
 
     @staticmethod
-    def connect_cvp():
-        """Connect to Cloudvision gRPC endpoint."""
-        return CloudvisionApi(
-            cvp_host=APP_SETTINGS["aristacv_cvp_host"],
-            cvp_port=APP_SETTINGS.get("aristacv_cvp_port", "8443"),
-            verify=APP_SETTINGS["aristacv_verify"],
-            username=APP_SETTINGS["aristacv_cvp_user"],
-            password=APP_SETTINGS["aristacv_cvp_password"],
-            cvp_token=APP_SETTINGS["aristacv_cvp_token"],
-        )
+    def connect_cvp(config: CloudVisionAppConfig):
+        """Connect to CloudVision gRPC endpoint."""
+        return CloudvisionApi(config)
 
     @classmethod
     def create(cls, diffsync, ids, attrs):
         """Create a user tag in cvp."""
-        cvp = cls.connect_cvp()
+        config: CloudVisionAppConfig = diffsync.job.app_config  # type: ignore
+        # TBD: Isn't this a performance bottleneck? We are connecting to CVP for each operation.
+        cvp = cls.connect_cvp(config)
         cvp.create_tag(ids["name"], attrs["value"])
         # Create mapping from device_name to CloudVision device_id
         device_ids = {dev["hostname"]: dev["device_id"] for dev in cvp.get_devices()}
@@ -159,7 +154,9 @@ def create(cls, diffsync, ids, attrs):
 
     def update(self, attrs):
         """Update user tag in cvp."""
-        cvp = self.connect_cvp()
+        config: CloudVisionAppConfig = self.diffsync.job.app_config  # type: ignore
+        # TBD: Isn't this a performance bottleneck? We are connecting to CVP for each operation.
+        cvp = self.connect_cvp(config)
         remove = set(self.device_name) - set(attrs["devices"])
         add = set(attrs["devices"]) - set(self.device_name)
         # Create mapping from device_name to CloudVision device_id
@@ -180,7 +177,9 @@ def update(self, attrs):
 
     def delete(self):
         """Delete user tag applied to devices in cvp."""
-        cvp = self.connect_cvp()
+        config: CloudVisionAppConfig = self.diffsync.job.app_config  # type: ignore
+        # TBD: Isn't this performance bottleneck? We are connecting to CVP for each operation.
+        cvp = self.connect_cvp(config)
         device_ids = {dev["hostname"]: dev["device_id"] for dev in cvp.get_devices()}
         for device in self.device_name:
             cvp.remove_tag_from_device(device_ids[device], self.name, self.value)