feat: update data model

Makefile

@@ -17,6 +17,11 @@ docker-compose-netbox-plugin-test:
 	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run -u root --rm netbox ./manage.py test --keepdb netbox_diode_plugin
 	@$(MAKE) docker-compose-netbox-plugin-down
 
+.PHONY: docker-compose-netbox-plugin-test-ff
+docker-compose-netbox-plugin-test-ff:
+	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run -u root --rm netbox ./manage.py test --failfast --keepdb netbox_diode_plugin
+	@$(MAKE) docker-compose-netbox-plugin-down
+
 .PHONY: docker-compose-netbox-plugin-test-cover
 docker-compose-netbox-plugin-test-cover:
 	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run --rm -u root -e COVERAGE_FILE=/opt/netbox/netbox/coverage/.coverage netbox sh -c "coverage run --source=netbox_diode_plugin --omit=*/migrations/* ./manage.py test --keepdb netbox_diode_plugin && coverage xml -o /opt/netbox/netbox/coverage/report.xml && coverage report -m | tee /opt/netbox/netbox/coverage/report.txt"

docker/netbox/configuration/configuration.py

@@ -44,9 +44,12 @@ def _environ_get_and_map(variable_name: str, default: str | None = None,
     return map_fn(env_value)
 
 
-_AS_BOOL = lambda value: value.lower() == 'true'
-_AS_INT = lambda value: int(value)
-_AS_LIST = lambda value: list(filter(None, value.split(' ')))
+def _AS_BOOL(value):
+    return value.lower() == 'true'
+def _AS_INT(value):
+    return int(value)
+def _AS_LIST(value):
+    return list(filter(None, value.split(' ')))
 
 _BASE_DIR = dirname(dirname(abspath(__file__)))
 

docker/netbox/env/netbox.env

@@ -41,3 +41,4 @@ DIODE_TO_NETBOX_API_KEY=1368dbad13e418d5a443d93cf255edde03a2a754
 NETBOX_TO_DIODE_API_KEY=1e99338b8cab5fc637bc55f390bda1446f619c42
 DIODE_API_KEY=5a52c45ee8231156cb620d193b0291912dd15433
 BASE_PATH=netbox/
+DEBUG=True

docker/netbox/local_settings.py

@@ -1,4 +1,5 @@
 from netbox_branching.utilities import DynamicSchemaDict
+
 from .configuration import DATABASE
 
 # Wrap DATABASES with DynamicSchemaDict for dynamic schema support
@@ -9,4 +10,4 @@
 # Employ our custom database router
 DATABASE_ROUTERS = [
     'netbox_branching.database.BranchAwareRouter',
-]
+]

netbox_diode_plugin/api/applier.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# Copyright 2024 NetBox Labs Inc
+"""Diode NetBox Plugin - API - Applier."""
+
+
+import logging
+
+from django.apps import apps
+from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import models
+from rest_framework.exceptions import ValidationError as ValidationError
+
+from .common import NON_FIELD_ERRORS, Change, ChangeSet, ChangeSetException, ChangeSetResult, ChangeType
+from .plugin_utils import get_object_type_model, legal_fields
+from .supported_models import get_serializer_for_model
+
+logger = logging.getLogger(__name__)
+
+
+def apply_changeset(change_set: ChangeSet) -> ChangeSetResult:
+    """Apply a change set."""
+    _validate_change_set(change_set)
+
+    created = {}
+    for i, change in enumerate(change_set.changes):
+        change_type = change.change_type
+        object_type = change.object_type
+
+        if change_type == ChangeType.NOOP.value:
+            continue
+
+        try:
+            model_class = get_object_type_model(object_type)
+            data = _pre_apply(model_class, change, created)
+            _apply_change(data, model_class, change, created)
+        except ValidationError as e:
+            raise _err_from_validation_error(e, f"changes[{i}]")
+        except ObjectDoesNotExist:
+            raise _err(f"{object_type} with id {change.object_id} does not exist", f"changes[{i}]", "object_id")
+        # ConstraintViolationError ?
+        # ...
+
+    return ChangeSetResult(
+        id=change_set.id,
+    )
+
+def _apply_change(data: dict, model_class: models.Model, change: Change, created: dict):
+    serializer_class = get_serializer_for_model(model_class)
+    change_type = change.change_type
+    if change_type == ChangeType.CREATE.value:
+        serializer = serializer_class(data=data)
+        serializer.is_valid(raise_exception=True)
+        instance = serializer.save()
+        created[change.ref_id] = instance
+
+    elif change_type == ChangeType.UPDATE.value:
+        if object_id := change.object_id:
+            instance = model_class.objects.get(id=object_id)
+            serializer = serializer_class(instance, data=data, partial=True)
+            serializer.is_valid(raise_exception=True)
+            serializer.save()
+        # create and update in a same change set
+        elif change.ref_id and (instance := created[change.ref_id]):
+            serializer = serializer_class(instance, data=data, partial=True)
+            serializer.is_valid(raise_exception=True)
+            serializer.save()
+
+def _pre_apply(model_class: models.Model, change: Change, created: dict):
+    data = change.data.copy()
+
+    # resolve foreign key references to new objects
+    for ref_field in change.new_refs:
+        if isinstance(data[ref_field], (list, tuple)):
+            ref_list = []
+            for ref in data[ref_field]:
+                if isinstance(ref, str):
+                    ref_list.append(created[ref].pk)
+                elif isinstance(ref, int):
+                    ref_list.append(ref)
+            data[ref_field] = ref_list
+        else:
+            data[ref_field] = created[data[ref_field]].pk
+
+    # ignore? fields that are not in the data model (error?)
+    allowed_fields = legal_fields(model_class)
+    for key in list(data.keys()):
+        if key not in allowed_fields:
+            logger.warning(f"Field {key} is not in the diode data model, ignoring.")
+            data.pop(key)
+
+    return data
+
+def _validate_change_set(change_set: ChangeSet):
+    if not change_set.id:
+        raise _err("Change set ID is required", "changeset","id")
+    if not change_set.changes:
+        raise _err("Changes are required", "changeset", "changes")
+
+    for i, change in enumerate(change_set.changes):
+        if change.object_id is None and change.ref_id is None:
+            raise _err("Object ID or Ref ID must be provided", f"changes[{i}]", NON_FIELD_ERRORS)
+        if change.change_type not in ChangeType:
+            raise _err(f"Unsupported change type '{change.change_type}'", f"changes[{i}]", "change_type")
+
+def _err(message, object_name, field):
+    return ChangeSetException(message, errors={object_name: {field: [message]}})
+
+def _err_from_validation_error(e, object_name):
+    errors = {}
+    if e.detail:
+        if isinstance(e.detail, dict):
+            errors[object_name] = e.detail
+        elif isinstance(e.detail, (list, tuple)):
+            errors[object_name] = {
+                NON_FIELD_ERRORS: e.detail
+            }
+        else:
+            errors[object_name] = {
+                NON_FIELD_ERRORS: [e.detail]
+            }
+    return ChangeSetException("validation error", errors=errors)

netbox_diode_plugin/api/common.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# Copyright 2025 NetBox Labs Inc
+"""Diode NetBox Plugin - API - Common types and utilities."""
+
+import logging
+import uuid
+from collections import defaultdict
+from dataclasses import dataclass, field
+from enum import Enum
+
+from django.apps import apps
+from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
+from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ValidationError
+from django.db import models
+from rest_framework import status
+
+logger = logging.getLogger("netbox.diode_data")
+
+NON_FIELD_ERRORS = "__all__"
+
+@dataclass
+class UnresolvedReference:
+    """unresolved reference to an object."""
+
+    object_type: str
+    uuid: str
+
+    def __str__(self):
+        """String representation of the unresolved reference."""
+        return f"new_object:{self.object_type}:{self.uuid}"
+
+    def __eq__(self, other):
+        """Equality operator."""
+        if not isinstance(other, UnresolvedReference):
+            return False
+        return self.object_type == other.object_type and self.uuid == other.uuid
+
+    def __hash__(self):
+        """Hash function."""
+        return hash((self.object_type, self.uuid))
+
+    def __lt__(self, other):
+        """Less than operator."""
+        return self.object_type < other.object_type or (self.object_type == other.object_type and self.uuid < other.uuid)
+
+
+class ChangeType(Enum):
+    """Change type enum."""
+
+    CREATE = "create"
+    UPDATE = "update"
+    NOOP = "noop"
+
+
+@dataclass
+class Change:
+    """A change to a model instance."""
+
+    change_type: ChangeType
+    object_type: str
+    object_id: int | None = field(default=None)
+    object_primary_value: str | None = field(default=None)
+    ref_id: str | None = field(default=None)
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    before: dict | None = field(default=None)
+    data: dict | None = field(default=None)
+    new_refs: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        """Convert the change to a dictionary."""
+        return {
+            "id": self.id,
+            "change_type": self.change_type.value,
+            "object_type": self.object_type,
+            "object_id": self.object_id,
+            "ref_id": self.ref_id,
+            "object_primary_value": self.object_primary_value,
+            "before": self.before,
+            "data": self.data,
+            "new_refs": self.new_refs,
+        }
+
+
+@dataclass
+class ChangeSet:
+    """A set of changes to a model instance."""
+
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    changes: list[Change] = field(default_factory=list)
+    branch: dict[str, str] | None = field(default=None)  # {"id": str, "name": str}
+
+    def to_dict(self) -> dict:
+        """Convert the change set to a dictionary."""
+        return {
+            "id": self.id,
+            "changes": [change.to_dict() for change in self.changes],
+            "branch": self.branch,
+        }
+
+    def validate(self) -> dict[str, list[str]]:
+        """Validate basics of the change set data."""
+        errors = defaultdict(dict)
+
+        for change in self.changes:
+            model = apps.get_model(change.object_type)
+
+            change_data = change.data.copy()
+            if change.before:
+                change_data.update(change.before)
+
+            excluded_relation_fields, rel_errors = self._validate_relations(change_data, model)
+            if rel_errors:
+                errors[change.object_type] = rel_errors
+
+            try:
+                instance = model(**change_data)
+                instance.clean_fields(exclude=excluded_relation_fields)
+            except ValidationError as e:
+                errors[change.object_type].update(e.error_dict)
+
+        return errors or None
+
+    def _validate_relations(self, change_data: dict, model: models.Model) -> tuple[list[str], dict]:
+        # check that there is some value for every required
+        # reference field, but don't validate the actual reference.
+        # the fields are removed from the change_data so that other
+        # fields can be validated by instantiating the model.
+        excluded_relation_fields = []
+        rel_errors = defaultdict(list)
+        for f in model._meta.get_fields():
+            if isinstance(f, (GenericRelation, GenericForeignKey)):
+                excluded_relation_fields.append(f.name)
+                continue
+            if not f.is_relation:
+                continue
+            field_name = f.name
+            excluded_relation_fields.append(field_name)
+
+            if hasattr(f, "related_model") and f.related_model == ContentType:
+                change_data.pop(field_name, None)
+                base_field = field_name[:-5]
+                excluded_relation_fields.append(base_field + "_id")
+                value = change_data.pop(base_field + "_id", None)
+            else:
+                value = change_data.pop(field_name, None)
+
+            if not f.null and not f.blank and not f.many_to_many:
+                # this field is a required relation...
+                if value is None:
+                    rel_errors[f.name].append(f"Field {f.name} is required")
+        return excluded_relation_fields, rel_errors
+
+
+@dataclass
+class ChangeSetResult:
+    """A result of applying a change set."""
+
+    id: str | None = field(default_factory=lambda: str(uuid.uuid4()))
+    change_set: ChangeSet | None = field(default=None)
+    errors: dict | None = field(default=None)
+
+    def to_dict(self) -> dict:
+        """Convert the result to a dictionary."""
+        if self.change_set:
+            return self.change_set.to_dict()
+
+        return {
+            "id": self.id,
+            "errors": self.errors,
+        }
+
+    def get_status_code(self) -> int:
+        """Get the status code for the result."""
+        return status.HTTP_200_OK if not self.errors else status.HTTP_400_BAD_REQUEST
+
+
+class ChangeSetException(Exception):
+    """ChangeSetException is raised when an error occurs while generating or applying a change set."""
+
+    def __init__(self, message, errors=None):
+        """Initialize the exception."""
+        super().__init__(message)
+        self.message = message
+        self.errors = errors or {}
+
+    def __str__(self):
+        """Return the string representation of the exception."""
+        if self.errors:
+            return f"{self.message}: {self.errors}"
+        return self.message