Skip to content

Commit

Permalink
Merge pull request #28 from nautobot/gfm-pk-cheating
Browse files Browse the repository at this point in the history 
Change of approach: use deterministic UUIDs for records imported into Nautobot
  • Loading branch information
@glennmatthews
glennmatthews committed Apr 5, 2021
2 parents 46809ca + 62c4813 commit 6fd1cfc
Show file tree
Hide file tree
Showing 20 changed files with 551 additions and 271 deletions.
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ The plugin is available as a Python package in PyPI and can be installed with pi
pip install nautobot-netbox-importer
```

> The plugin is compatible with Nautobot 1.0 and can handle JSON data exported from NetBox 2.10.3 through 2.10.5 at present.
> The plugin is compatible with Nautobot 1.0.0b3 and later and can handle JSON data exported from NetBox 2.10.x at present.
Once installed, the plugin needs to be enabled in your `nautobot_config.py`:

Expand All @@ -22,7 +22,7 @@ PLUGINS = ["nautobot_netbox_importer"]

### Getting a data export from NetBox

From the NetBox root directory, run the following command:
From the NetBox root directory, run the following command to produce a JSON file (here, `/tmp/netbox_data.json`) describing the contents of your NetBox database:

```shell
python netbox/manage.py dumpdata \
Expand All @@ -34,7 +34,7 @@ python netbox/manage.py dumpdata \

### Importing the data into Nautobot

From the Nautobot root directory, run `nautobot-server import_netbox_json <json_file> <netbox_version>`, for example `nautobot-server import_netbox_json /tmp/netbox_data.json 2.10.3`.
From within the Nautobot application environment, run `nautobot-server import_netbox_json <json_file> <netbox_version>`, for example `nautobot-server import_netbox_json /tmp/netbox_data.json 2.10.3`.

## Contributing

Expand Down
46 changes: 10 additions & 36 deletions nautobot_netbox_importer/diffsync/adapters/abstract.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
from uuid import UUID

from diffsync import Diff, DiffSync, DiffSyncFlags, DiffSyncModel
from diffsync.exceptions import ObjectAlreadyExists
from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound
from pydantic.error_wrappers import ValidationError
import structlog

import nautobot_netbox_importer.diffsync.models as n2nmodels
from nautobot_netbox_importer.diffsync.models.validation import netbox_pk_to_nautobot_pk


class N2NDiffSync(DiffSync):
Expand Down Expand Up @@ -121,6 +122,9 @@ class N2NDiffSync(DiffSync):
# The specific order of models below is constructed empirically, but basically attempts to place all models
# in sequence so that if model A has a hard dependency on a reference to model B, model B gets processed first.
#
# Note: with the latest changes in design for this plugin (using deterministic UUIDs in Nautobot to allow
# direct mapping of NetBox PKs to Nautobot PKs), this order is now far less critical than it was previously.
#

top_level = (
# "contenttype", Not synced, as these are hard-coded in NetBox/Nautobot
Expand All @@ -129,7 +133,7 @@ class N2NDiffSync(DiffSync):
"user", # Includes NetBox "userconfig" model as well
"objectpermission",
"token",
# "status", Not synced, as these are hard-coded in NetBox/Nautobot
# "status", Not synced, as these are hard-coded in NetBox and autogenerated in Nautobot
# Need Tenant and TenantGroup before we can populate Sites
"tenantgroup",
"tenant", # Not all Tenants belong to a TenantGroup
Expand Down Expand Up @@ -192,7 +196,6 @@ class N2NDiffSync(DiffSync):
# Interface/VMInterface -> Device/VirtualMachine (device)
# Interface comes after Device because it MUST have a Device to be created;
# IPAddress comes after Interface because we use the assigned_object as part of the IP's unique ID.
# We will fixup the Device->primary_ip reference in fixup_data_relations()
"ipaddress",
"cable",
"service",
Expand Down Expand Up @@ -231,41 +234,10 @@ def add(self, obj: DiffSyncModel):
self._data_by_pk[modelname][obj.pk] = obj
super().add(obj)

def fixup_data_relations(self):
"""Iterate once more over all models and fix up any leftover FK relations."""
for name in self.top_level:
instances = self.get_all(name)
if not instances:
self.logger.info("No instances to review", model=name)
else:
self.logger.info(f"Reviewing all {len(instances)} instances", model=name)
for diffsync_instance in instances:
for fk_field, target_name in diffsync_instance.fk_associations().items():
value = getattr(diffsync_instance, fk_field)
if not value:
continue
if "*" in target_name:
target_content_type_field = target_name[1:]
target_content_type = getattr(diffsync_instance, target_content_type_field)
target_name = target_content_type["model"]
target_class = getattr(self, target_name)
if "pk" in value:
new_value = self.get_fk_identifiers(diffsync_instance, target_class, value["pk"])
if isinstance(new_value, (UUID, int)):
self.logger.error(
"Still unable to resolve reference?",
source=diffsync_instance,
target=target_name,
pk=new_value,
)
else:
self.logger.debug(
"Replacing forward reference with identifiers", pk=value["pk"], identifiers=new_value
)
setattr(diffsync_instance, fk_field, new_value)

def get_fk_identifiers(self, source_object, target_class, pk):
"""Helper to load_record: given a class and a PK, get the identifiers of the given instance."""
if isinstance(pk, int):
pk = netbox_pk_to_nautobot_pk(target_class.get_type(), pk)
target_record = self.get_by_pk(target_class, pk)
if not target_record:
self.logger.debug(
Expand All @@ -283,6 +255,8 @@ def get_by_pk(self, obj, pk):
modelname = obj
else:
modelname = obj.get_type()
if pk not in self._data_by_pk[modelname]:
raise ObjectNotFound(f"PK {pk} not found in stored {modelname} instances")
return self._data_by_pk[modelname].get(pk)

def make_model(self, diffsync_model, data):
Expand Down
37 changes: 22 additions & 15 deletions nautobot_netbox_importer/diffsync/adapters/nautobot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import structlog

from .abstract import N2NDiffSync
from ..models.abstract import NautobotBaseModel


IGNORED_FIELD_CLASSES = (GenericRel, GenericForeignKey, models.ManyToManyRel, models.ManyToOneRel)
Expand All @@ -23,7 +24,7 @@ class NautobotDiffSync(N2NDiffSync):

logger = structlog.get_logger()

def load_model(self, diffsync_model, record):
def load_model(self, diffsync_model, record): # pylint: disable=too-many-branches
"""Instantiate the given DiffSync model class from the given Django record."""
data = {}

Expand All @@ -46,7 +47,8 @@ def load_model(self, diffsync_model, record):

# If we got here, the field is some sort of foreign-key reference(s).
if not value:
# It's a null reference though, so we don't need to do anything special with it.
# It's a null or empty list reference though, so we don't need to do anything special with it.
data[field.name] = value
continue

# What's the name of the model that this is a reference to?
Expand All @@ -69,16 +71,24 @@ def load_model(self, diffsync_model, record):
continue

if isinstance(value, list):
# This field is a one-to-many or many-to-many field, a list of foreign key references.
# For each foreign key, find the corresponding DiffSync record, and use its
# natural keys (identifiers) in the data in place of the foreign key value.
data[field.name] = [
self.get_fk_identifiers(diffsync_model, target_class, foreign_record.pk) for foreign_record in value
]
elif isinstance(value, (UUID, int)):
# Look up the DiffSync record corresponding to this foreign key,
# and store its natural keys (identifiers) in the data in place of the foreign key value.
data[field.name] = self.get_fk_identifiers(diffsync_model, target_class, value)
# This field is a one-to-many or many-to-many field, a list of object references.
if issubclass(target_class, NautobotBaseModel):
# Replace each object reference with its appropriate primary key value
data[field.name] = [foreign_record.pk for foreign_record in value]
else:
# Since the PKs of these built-in Django models may differ between NetBox and Nautobot,
# e.g., ContentTypes, replace each reference with the natural key (not PK) of the referenced model.
data[field.name] = [
self.get_by_pk(target_name, foreign_record.pk).get_identifiers() for foreign_record in value
]
elif isinstance(value, UUID):
# Standard Nautobot UUID foreign-key reference, no transformation needed.
data[field.name] = value
elif isinstance(value, int):
# Reference to a built-in model by its integer primary key.
# Since this may not be the same value between NetBox and Nautobot (e.g., ContentType references)
# replace the PK with the natural keys of the referenced model.
data[field.name] = self.get_by_pk(target_name, value).get_identifiers()
else:
self.logger.error(f"Invalid PK value {value}")
data[field.name] = None
Expand All @@ -95,7 +105,4 @@ def load(self):
for instance in diffsync_model.nautobot_model().objects.all():
self.load_model(diffsync_model, instance)

self.logger.info("Fixing up any previously unresolved object relations...")
self.fixup_data_relations()

self.logger.info("Data loading from Nautobot complete.")
37 changes: 23 additions & 14 deletions nautobot_netbox_importer/diffsync/adapters/netbox.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
"""DiffSync adapters for NetBox data dumps."""

import json
from uuid import UUID, uuid4
from uuid import uuid4

import structlog

from .abstract import N2NDiffSync
from ..models.abstract import NautobotBaseModel
from ..models.validation import netbox_pk_to_nautobot_pk


class NetBox210DiffSync(N2NDiffSync):
Expand All @@ -21,6 +23,7 @@ def __init__(self, *args, source_data=None, **kwargs):
def load_record(self, diffsync_model, record): # pylint: disable=too-many-branches
"""Instantiate the given model class from the given record."""
data = record["fields"].copy()
data["pk"] = record["pk"]

# Fixup fields that are actually foreign-key (FK) associations by replacing
# their FK ids with the DiffSync model unique-id fields.
Expand All @@ -42,7 +45,7 @@ def load_record(self, diffsync_model, record): # pylint: disable=too-many-branc
if target_name.startswith("*"):
target_content_type_field = target_name[1:]
target_content_type_pk = record["fields"][target_content_type_field]
if not isinstance(target_content_type_pk, int) and not isinstance(target_content_type_pk, str):
if not isinstance(target_content_type_pk, int):
self.logger.error(f"Invalid content-type PK value {target_content_type_pk}")
data[key] = None
continue
Expand All @@ -59,13 +62,22 @@ def load_record(self, diffsync_model, record): # pylint: disable=too-many-branc

if isinstance(data[key], list):
# This field is a one-to-many or many-to-many field, a list of foreign key references.
# For each foreign key, find the corresponding DiffSync record, and use its
# natural keys (identifiers) in the data in place of the foreign key value.
data[key] = [self.get_fk_identifiers(diffsync_model, target_class, pk) for pk in data[key]]
elif isinstance(data[key], (UUID, int)):
# Look up the DiffSync record corresponding to this foreign key,
# and store its natural keys (identifiers) in the data in place of the foreign key value.
data[key] = self.get_fk_identifiers(diffsync_model, target_class, data[key])
if issubclass(target_class, NautobotBaseModel):
# Replace each NetBox integer FK with the corresponding deterministic Nautobot UUID FK.
data[key] = [netbox_pk_to_nautobot_pk(target_name, pk) for pk in data[key]]
else:
# It's a base Django model such as ContentType or Group.
# Since we can't easily control its PK in Nautobot, use its natural key instead
data[key] = [self.get_by_pk(target_name, pk).get_identifiers() for pk in data[key]]
elif isinstance(data[key], int):
# Standard NetBox integer foreign-key reference
if issubclass(target_class, NautobotBaseModel):
# Replace the NetBox integer FK with the corresponding deterministic Nautobot UUID FK.
data[key] = netbox_pk_to_nautobot_pk(target_name, data[key])
else:
# It's a base Django model such as ContentType or Group.
# Since we can't easily control its PK in Nautobot, use its natural key instead
data[key] = self.get_by_pk(target_name, data[key]).get_identifiers()
else:
self.logger.error(f"Invalid PK value {data[key]}")
data[key] = None
Expand All @@ -89,11 +101,11 @@ def load_record(self, diffsync_model, record): # pylint: disable=too-many-branc
# see also models.abstract.ArrayField
for choice in json.loads(data["choices"]):
self.make_model(
self.customfieldchoice, {"pk": uuid4(), "field": {"name": data["name"]}, "value": choice}
self.customfieldchoice,
{"pk": uuid4(), "field": netbox_pk_to_nautobot_pk("customfield", record["pk"]), "value": choice},
)
del data["choices"]

data["pk"] = record["pk"]
return self.make_model(diffsync_model, data)

def load(self):
Expand All @@ -111,9 +123,6 @@ def load(self):
if record["model"] == content_type_label:
self.load_record(diffsync_model, record)

self.logger.info("Fixing up any previously unresolved object relations...")
self.fixup_data_relations()

self.logger.info("Data loading from NetBox source data complete.")
# Discard the source data to free up memory
self.source_data = None
Loading

0 comments on commit 6fd1cfc

Please sign in to comment.