diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 00000000..e6ec0297
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,23 @@
+---
+name: 🐛 Bug Report
+about: Report a reproducible bug in the current release of DSync
+---
+
+### Environment
+* DSync version: <!-- Example: 0.1.0 -->
+* Python version <!-- Example: 3.7.7 -->
+
+<!-- What happened instead? -->
+### Observed Behavior
+
+<!-- What did you expect to happen? -->
+### Expected Behavior
+
+<!--
+ Describe in detail the exact steps that someone else can take to reproduce
+ this bug using the current release.
+-->
+### Steps to Reproduce
+1.
+2.
+3.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 00000000..00f22cf5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,21 @@
+---
+name: ✨ Feature Request
+about: Propose a new feature or enhancement
+---
+
+### Environment
+* DSync version: <!-- Example: 0.1.0 -->
+
+<!--
+ Describe in detail the new functionality you are proposing.
+-->
+### Proposed Functionality
+
+
+<!--
+ Convey an example use case for your proposed feature. Write from the
+ perspective of a user who would benefit from the proposed
+ functionality and describe how.
+--->
+### Use Case
+
diff --git a/dsync/__init__.py b/dsync/__init__.py
index a73548c2..03037f52 100644
--- a/dsync/__init__.py
+++ b/dsync/__init__.py
@@ -15,10 +15,10 @@
from collections.abc import Iterable as ABCIterable, Mapping as ABCMapping
import enum
from inspect import isclass
-import logging
from typing import ClassVar, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Type, Union
from pydantic import BaseModel
+import structlog # type: ignore
from .diff import Diff, DiffElement
from .utils import intersection
@@ -32,7 +32,7 @@
ObjectNotFound,
)
-logger = logging.getLogger(__name__)
+_logger = structlog.get_logger()
class DSyncFlags(enum.Flag):
@@ -321,6 +321,9 @@ class DSync:
# modelname1 = MyModelClass1
# modelname2 = MyModelClass2
+ type: ClassVar[Optional[str]] = None
+ """Type of the object, will default to the name of the class if not provided."""
+
top_level: ClassVar[List[str]] = []
"""List of top-level modelnames to begin from when diffing or synchronizing."""
@@ -330,12 +333,20 @@ class DSync:
`self._data[modelname][unique_id] == model_instance`
"""
- def __init__(self):
+ def __init__(self, name=None):
"""Generic initialization function.
Subclasses should be careful to call super().__init__() if they override this method.
"""
self._data = defaultdict(dict)
+ self._log = _logger.new(dsync=self)
+
+ # If the type is not defined, use the name of the class as the default value
+ if self.type is None:
+ self.type = self.__class__.__name__
+
+ # If the name has not been provided, use the type as the name
+ self.name = name if name else self.type
def __init_subclass__(cls):
"""Validate that references to specific DSyncModels use the correct modelnames.
@@ -349,6 +360,15 @@ def __init_subclass__(cls):
f'Incorrect field name - {value.__name__} has type name "{value.get_type()}", not "{name}"'
)
+ def __str__(self):
+ """String representation of a DSync."""
+ if self.type != self.name:
+ return f'{self.type} "{self.name}"'
+ return self.type
+
+ def __repr__(self):
+ return f"<{str(self)}>"
+
def load(self):
"""Load all desired data from whatever backend data source into this instance."""
# No-op in this generic class
@@ -365,12 +385,13 @@ def sync_from(self, source: "DSync", diff_class: Type[Diff] = Diff, flags: DSync
diff_class (class): Diff or subclass thereof to use to calculate the diffs to use for synchronization
flags (DSyncFlags): Flags influencing the behavior of this sync.
"""
+ log = self._log.bind(src=source, dst=self, flags=flags).unbind("dsync")
diff = self.diff_from(source, diff_class=diff_class, flags=flags)
- logger.info("Beginning sync")
+ log.info("Beginning sync")
for child in diff.get_children():
- self._sync_from_diff_element(child, flags=flags)
- logger.info("Sync complete")
+ self._sync_from_diff_element(child, flags=flags, logger=log)
+ log.info("Sync complete")
def sync_to(self, target: "DSync", diff_class: Type[Diff] = Diff, flags: DSyncFlags = DSyncFlags.NONE):
"""Synchronize data from the current DSync object into the given target DSync object.
@@ -383,7 +404,11 @@ def sync_to(self, target: "DSync", diff_class: Type[Diff] = Diff, flags: DSyncFl
target.sync_from(self, diff_class=diff_class, flags=flags)
def _sync_from_diff_element(
- self, element: DiffElement, flags: DSyncFlags = DSyncFlags.NONE, parent_model: DSyncModel = None,
+ self,
+ element: DiffElement,
+ flags: DSyncFlags = DSyncFlags.NONE,
+ logger: structlog.BoundLogger = None,
+ parent_model: DSyncModel = None,
):
"""Synchronize a given DiffElement (and its children, if any) into this DSync.
@@ -392,49 +417,53 @@ def _sync_from_diff_element(
Args:
element: DiffElement to synchronize diffs from
flags (DSyncFlags): Flags influencing the behavior of this sync.
+ logger: Parent logging context
parent_model: Parent object to update (`add_child`/`remove_child`) if the sync creates/deletes an object.
"""
# pylint: disable=too-many-branches
# GFM: I made a few attempts at refactoring this to reduce the branching, but found that it was less readable.
# So let's live with the slightly too high number of branches (14/12) for now.
+ log = logger or self._log
object_class = getattr(self, element.type)
obj = self.get(object_class, element.keys)
- attrs = (
- {attr_key: element.source_attrs[attr_key] for attr_key in element.get_attrs_keys()}
- if element.source_attrs is not None
- else {}
+ # Get the attributes that actually differ between source and dest
+ diffs = element.get_attrs_diffs()
+ log = log.bind(
+ action=element.action,
+ model=object_class.get_type(),
+ unique_id=object_class.create_unique_id(**element.keys),
+ diffs=diffs,
)
try:
if element.action == "create":
+ log.debug("Attempting object creation")
if obj:
raise ObjectNotCreated(f"Failed to create {object_class.get_type()} {element.keys} - it exists!")
- logger.info("Creating %s %s (%s)", object_class.get_type(), element.keys, attrs)
- obj = object_class.create(dsync=self, ids=element.keys, attrs=attrs)
+ obj = object_class.create(dsync=self, ids=element.keys, attrs={key: diffs[key]["src"] for key in diffs})
+ log.info("Created successfully", status="success")
elif element.action == "update":
+ log.debug("Attempting object update")
if not obj:
raise ObjectNotUpdated(f"Failed to update {object_class.get_type()} {element.keys} - not found!")
- logger.info("Updating %s %s with %s", object_class.get_type(), element.keys, attrs)
- obj = obj.update(attrs=attrs)
+ obj = obj.update(attrs={key: diffs[key]["src"] for key in diffs})
+ log.info("Updated successfully", status="success")
elif element.action == "delete":
+ log.debug("Attempting object deletion")
if not obj:
raise ObjectNotDeleted(f"Failed to delete {object_class.get_type()} {element.keys} - not found!")
- logger.info("Deleting %s %s", object_class.get_type(), element.keys)
obj = obj.delete()
+ log.info("Deleted successfully", status="success")
except ObjectCrudException as exception:
- logger.error(
- "Error during %s of %s %s (%s): %s",
- element.action,
- object_class.get_type(),
- element.keys,
- attrs,
- exception,
- )
+ log.error(str(exception), status="error")
if not flags & DSyncFlags.CONTINUE_ON_FAILURE:
raise
+ else:
+ if obj is None:
+ log.warning("Non-fatal failure encountered", status="failure")
if obj is None:
- logger.warning("Not syncing children of %s %s", element.type, element.keys)
+ log.warning("Not syncing children")
return
if element.action == "create":
@@ -447,7 +476,7 @@ def _sync_from_diff_element(
parent_model.remove_child(obj)
for child in element.get_children():
- self._sync_from_diff_element(child, flags=flags, parent_model=obj)
+ self._sync_from_diff_element(child, flags=flags, parent_model=obj, logger=logger)
# ------------------------------------------------------------------------------
# Diff calculation and construction
@@ -461,20 +490,25 @@ def diff_from(self, source: "DSync", diff_class: Type[Diff] = Diff, flags: DSync
diff_class (class): Diff or subclass thereof to use for diff calculation and storage.
flags (DSyncFlags): Flags influencing the behavior of this diff operation.
"""
- logger.info("Beginning diff")
+ log = self._log.bind(src=source, dst=self, flags=flags).unbind("dsync")
+ log.info("Beginning diff")
diff = diff_class()
for obj_type in intersection(self.top_level, source.top_level):
diff_elements = self._diff_objects(
- source=source.get_all(obj_type), dest=self.get_all(obj_type), source_root=source, flags=flags,
+ source=source.get_all(obj_type),
+ dest=self.get_all(obj_type),
+ source_root=source,
+ flags=flags,
+ logger=log,
)
for diff_element in diff_elements:
diff.add(diff_element)
# Notify the diff that it has been fully populated, in case it wishes to print, save to a file, etc.
- logger.info("Diff complete")
+ log.info("Diff complete")
diff.complete()
return diff
@@ -488,12 +522,13 @@ def diff_to(self, target: "DSync", diff_class: Type[Diff] = Diff, flags: DSyncFl
"""
return target.diff_from(self, diff_class=diff_class, flags=flags)
- def _diff_objects(
+ def _diff_objects( # pylint: disable=too-many-arguments
self,
source: Iterable[DSyncModel],
dest: Iterable[DSyncModel],
source_root: "DSync",
flags: DSyncFlags = DSyncFlags.NONE,
+ logger: structlog.BoundLogger = None,
) -> List[DiffElement]:
"""Generate a list of DiffElement between the given lists of objects.
@@ -504,6 +539,7 @@ def _diff_objects(
dest: DSyncModel instances retrieved from this DSync instance
source_root (DSync): The other DSync object being diffed against (owner of the `source` models, if any)
flags (DSyncFlags): Flags influencing the behavior of this diff operation.
+ logger: Parent logging context
Raises:
TypeError: if the source and dest args are not the same type, or if that type is unsupported
@@ -527,26 +563,35 @@ def _diff_objects(
self._validate_objects_for_diff(combined_dict)
for uid in combined_dict:
+ log = logger or self._log
src_obj, dst_obj = combined_dict[uid]
-
- if flags & DSyncFlags.SKIP_UNMATCHED_SRC and not dst_obj:
- logger.debug("Skipping unmatched source object {src_obj}")
- continue
- if flags & DSyncFlags.SKIP_UNMATCHED_DST and not src_obj:
- logger.debug("Skipping unmatched dest object {dst_obj}")
- continue
-
+ if not src_obj and not dst_obj:
+ # Should never happen
+ raise RuntimeError(f"UID {uid} is in combined_dict but has neither src_obj nor dst_obj??")
if src_obj:
+ log = log.bind(model=src_obj.get_type(), unique_id=src_obj.get_unique_id())
+ if flags & DSyncFlags.SKIP_UNMATCHED_SRC and not dst_obj:
+ log.debug("Skipping unmatched source object")
+ continue
diff_element = DiffElement(
- obj_type=src_obj.get_type(), name=src_obj.get_shortname(), keys=src_obj.get_identifiers()
+ obj_type=src_obj.get_type(),
+ name=src_obj.get_shortname(),
+ keys=src_obj.get_identifiers(),
+ source_name=source_root.name,
+ dest_name=self.name,
)
elif dst_obj:
+ log = log.bind(model=dst_obj.get_type(), unique_id=dst_obj.get_unique_id())
+ if flags & DSyncFlags.SKIP_UNMATCHED_DST and not src_obj:
+ log.debug("Skipping unmatched dest object")
+ continue
diff_element = DiffElement(
- obj_type=dst_obj.get_type(), name=dst_obj.get_shortname(), keys=dst_obj.get_identifiers()
+ obj_type=dst_obj.get_type(),
+ name=dst_obj.get_shortname(),
+ keys=dst_obj.get_identifiers(),
+ source_name=source_root.name,
+ dest_name=self.name,
)
- else:
- # Should be unreachable
- raise RuntimeError(f"UID {uid} is in combined_dict but has neither src_obj nor dst_obj??")
if src_obj:
diff_element.add_attrs(source=src_obj.get_attrs(), dest=None)
@@ -554,7 +599,7 @@ def _diff_objects(
diff_element.add_attrs(source=None, dest=dst_obj.get_attrs())
# Recursively diff the children of src_obj and dst_obj and attach the resulting diffs to the diff_element
- self._diff_child_objects(diff_element, src_obj, dst_obj, source_root, flags)
+ self._diff_child_objects(diff_element, src_obj, dst_obj, source_root, flags=flags, logger=logger)
diffs.append(diff_element)
@@ -589,6 +634,7 @@ def _diff_child_objects( # pylint: disable=too-many-arguments
dst_obj: Optional[DSyncModel],
source_root: "DSync",
flags: DSyncFlags,
+ logger: structlog.BoundLogger,
):
"""For all children of the given DSyncModel pair, diff them recursively, adding diffs to the given diff_element.
@@ -620,7 +666,7 @@ def _diff_child_objects( # pylint: disable=too-many-arguments
dst_objs = self.get_by_uids(getattr(dst_obj, child_fieldname), child_type) if dst_obj else []
for child_diff_element in self._diff_objects(
- source=src_objs, dest=dst_objs, source_root=source_root, flags=flags
+ source=src_objs, dest=dst_objs, source_root=source_root, flags=flags, logger=logger,
):
diff_element.add_child(child_diff_element)
diff --git a/dsync/diff.py b/dsync/diff.py
index 1838dccc..d53905ce 100644
--- a/dsync/diff.py
+++ b/dsync/diff.py
@@ -14,7 +14,7 @@
"""
from functools import total_ordering
-from typing import Iterator, Iterable, Optional
+from typing import Any, Iterator, Iterable, Mapping, Optional
from .exceptions import ObjectAlreadyExists
from .utils import intersection, OrderedDefaultDict
@@ -108,10 +108,12 @@ def print_detailed(self, indent: int = 0):
@total_ordering
-class DiffElement:
+class DiffElement: # pylint: disable=too-many-instance-attributes
"""DiffElement object, designed to represent a single item/object that may or may not have any diffs."""
- def __init__(self, obj_type: str, name: str, keys: dict):
+ def __init__(
+ self, obj_type: str, name: str, keys: dict, source_name: str = "source", dest_name: str = "dest"
+ ): # pylint: disable=too-many-arguments
"""Instantiate a DiffElement.
Args:
@@ -119,6 +121,8 @@ def __init__(self, obj_type: str, name: str, keys: dict):
name (str): Human-readable name of the object being described, as in DSyncModel.get_shortname().
This name must be unique within the context of the Diff that is the direct parent of this DiffElement.
keys (dict): Primary keys and values uniquely describing this object, as in DSyncModel.get_identifiers().
+ source_name (str): Name of the source DSync object
+ dest_name (str): Name of the destination DSync object
"""
if not isinstance(obj_type, str):
raise ValueError(f"obj_type must be a string (not {type(obj_type)})")
@@ -129,6 +133,8 @@ def __init__(self, obj_type: str, name: str, keys: dict):
self.type = obj_type
self.name = name
self.keys = keys
+ self.source_name = source_name
+ self.dest_name = dest_name
# Note: *_attrs == None if no target object exists; it'll be an empty dict if it exists but has no _attributes
self.source_attrs: Optional[dict] = None
self.dest_attrs: Optional[dict] = None
@@ -159,7 +165,7 @@ def __eq__(self, other):
def __str__(self):
"""Basic string representation of a DiffElement."""
- return f"{self.type} : {self.name} : {self.keys} : {self.source_attrs} : {self.dest_attrs}"
+ return f'{self.type} "{self.name}" : {self.keys} : {self.source_name} → {self.dest_name} : {self.get_attrs_diffs()}'
@property
def action(self) -> Optional[str]:
@@ -206,6 +212,26 @@ def get_attrs_keys(self) -> Iterable[str]:
return self.source_attrs.keys()
return []
+ # The below would be more accurate but typing.Literal is only in Python 3.8 and later
+ # def get_attrs_diffs(self) -> Mapping[str, Mapping[Literal["src", "dst"], Any]]:
+ def get_attrs_diffs(self) -> Mapping[str, Mapping[str, Any]]:
+ """Get the dict of actual attribute diffs between source_attrs and dest_attrs.
+
+ Returns:
+ dict: of the form `{key: {src: <value>, dst: <value>}, key2: ...}`
+ """
+ if self.source_attrs is not None and self.dest_attrs is not None:
+ return {
+ key: dict(src=self.source_attrs[key], dst=self.dest_attrs[key])
+ for key in self.get_attrs_keys()
+ if self.source_attrs[key] != self.dest_attrs[key]
+ }
+ if self.source_attrs is None and self.dest_attrs is not None:
+ return {key: dict(src=None, dst=self.dest_attrs[key]) for key in self.get_attrs_keys()}
+ if self.source_attrs is not None and self.dest_attrs is None:
+ return {key: dict(src=self.source_attrs[key], dst=None) for key in self.get_attrs_keys()}
+ return {}
+
def add_child(self, element: "DiffElement"):
"""Attach a child object of type DiffElement.
@@ -250,14 +276,13 @@ def print_detailed(self, indent: int = 0):
margin = " " * indent
if self.source_attrs is None:
- print(f"{margin}{self.type}: {self.name} MISSING in SOURCE")
+ print(f"{margin}{self.type}: {self.name} MISSING in {self.source_name}")
elif self.dest_attrs is None:
- print(f"{margin}{self.type}: {self.name} MISSING in DEST")
+ print(f"{margin}{self.type}: {self.name} MISSING in {self.dest_name}")
else:
print(f"{margin}{self.type}: {self.name}")
# Only print attrs that have meaning in both source and dest
- for attr in self.get_attrs_keys():
- if self.source_attrs[attr] != self.dest_attrs[attr]:
- print(f"{margin} {attr} S({self.source_attrs[attr]}) D({self.dest_attrs[attr]})")
+ for attr, item in self.get_attrs_diffs().items():
+ print(f"{margin} {attr} {self.source_name}({item.get('src')}) {self.dest_name}({item.get('dst')})")
self.child_diff.print_detailed(indent + 2)
diff --git a/dsync/logging.py b/dsync/logging.py
new file mode 100644
index 00000000..c2bed410
--- /dev/null
+++ b/dsync/logging.py
@@ -0,0 +1,37 @@
+"""Helpful APIs for setting up DSync logging."""
+
+import logging
+
+import structlog # type: ignore
+
+
+def enable_console_logging(verbosity=0):
+ """Enable formatted logging to console with the specified verbosity.
+
+ See https://www.structlog.org/en/stable/development.html as a reference
+
+ Args:
+ verbosity (int): 0 for WARNING logs, 1 for INFO logs, 2 for DEBUG logs
+ """
+ if verbosity == 0:
+ logging.basicConfig(format="%(message)s", level=logging.WARNING)
+ elif verbosity == 1:
+ logging.basicConfig(format="%(message)s", level=logging.INFO)
+ else:
+ logging.basicConfig(format="%(message)s", level=logging.DEBUG)
+
+ structlog.configure(
+ processors=[
+ structlog.stdlib.filter_by_level, # <-- added
+ structlog.stdlib.add_logger_name,
+ structlog.stdlib.add_log_level,
+ structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S"),
+ structlog.processors.StackInfoRenderer(),
+ structlog.processors.format_exc_info,
+ structlog.dev.ConsoleRenderer(),
+ ],
+ context_class=dict,
+ logger_factory=structlog.stdlib.LoggerFactory(),
+ wrapper_class=structlog.stdlib.BoundLogger,
+ cache_logger_on_first_use=True,
+ )
diff --git a/examples/example1/README.md b/examples/example1/README.md
index cdf43dc7..89717039 100644
--- a/examples/example1/README.md
+++ b/examples/example1/README.md
@@ -22,6 +22,14 @@ c = BackendC()
c.load()
```
+Configure verbosity of DSync's structured logging to console; the default is full verbosity (all logs including debugging)
+```python
+from dsync.logging import enable_console_logging
+enable_console_logging(verbosity=0) # Show WARNING and ERROR logs only
+# enable_console_logging(verbosity=1) # Also include INFO logs
+# enable_console_logging(verbosity=2) # Also include DEBUG logs
+```
+
Show the differences between A and B
```python
diff_a_b = a.diff_to(b)
@@ -46,4 +54,4 @@ diff_a_b = a.diff_to(b)
diff_a_b.print_detailed()
```
-> In the Device model, the role is not defined as an attribute so it's not shown when we are comparing the different objects, even if the value is different.
+> In the Device model, the `site_name` and `role` are not included in the `_attributes`, so they are not shown when we are comparing the different objects, even if the value is different.
diff --git a/examples/example1/backend_a.py b/examples/example1/backend_a.py
index 67ccb4fc..ac0a9f8d 100644
--- a/examples/example1/backend_a.py
+++ b/examples/example1/backend_a.py
@@ -11,7 +11,7 @@
},
"sfo": {
"sfo-spine1": {"role": "spine", "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"}},
- "sfo-spine2": {"role": "spine", "interfaces": {"eth0": "TBD", "eth1": "ddd"}},
+ "sfo-spine2": {"role": "spine", "interfaces": {"eth0": "TBD", "eth1": "ddd", "eth2": "Interface 2"}},
},
}
@@ -25,6 +25,8 @@ class BackendA(DSync):
top_level = ["site"]
+ type = "Backend A"
+
nb = None
def load(self):
diff --git a/examples/example1/backend_b.py b/examples/example1/backend_b.py
index dd3553e7..68aa548c 100644
--- a/examples/example1/backend_b.py
+++ b/examples/example1/backend_b.py
@@ -5,6 +5,10 @@
from models import Site, Device, Interface
DATA = {
+ "atl": {
+ "atl-spine1": {"role": "spine", "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"}},
+ "atl-spine2": {"role": "spine", "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"}},
+ },
"nyc": {
"nyc-spine1": {"role": "spine", "interfaces": {"eth0": "Interface 0/0", "eth1": "Interface 1"}},
"nyc-spine2": {"role": "spine", "interfaces": {"eth0": "Interface 0", "eth1": "Interface 1"}},
diff --git a/examples/example1/main.py b/examples/example1/main.py
index 8fabf85f..33fffc47 100644
--- a/examples/example1/main.py
+++ b/examples/example1/main.py
@@ -1,4 +1,9 @@
"""Main executable for DSync "example1"."""
+# pylint: disable=wrong-import-order
+
+import argparse
+
+from dsync.logging import enable_console_logging
from backend_a import BackendA
from backend_b import BackendB
@@ -21,22 +26,31 @@ def order_children_site(cls, children):
def main():
"""Demonstrate DSync behavior using the example backends provided."""
- # pylint: disable=invalid-name
+ parser = argparse.ArgumentParser("example1")
+ parser.add_argument("--verbosity", "-v", default=0, action="count")
+ args = parser.parse_args()
+ enable_console_logging(verbosity=args.verbosity)
- a = BackendA()
- a.load()
+ print("Initializing and loading Backend A...")
+ backend_a = BackendA(name="Backend-A")
+ backend_a.load()
- b = BackendB()
- b.load()
+ print("Initializing and loading Backend B...")
+ backend_b = BackendB(name="Backend-B")
+ backend_b.load()
- c = BackendC()
- c.load()
+ print("Initializing and loading Backend C...")
+ backend_c = BackendC()
+ backend_c.load()
- diff_a_b = a.diff_to(b, diff_class=MyDiff)
+ print("Getting diffs from Backend A to Backend B...")
+ diff_a_b = backend_a.diff_to(backend_b, diff_class=MyDiff)
diff_a_b.print_detailed()
- a.sync_to(b)
- a.diff_to(b).print_detailed()
+ print("Syncing changes from Backend A to Backend B...")
+ backend_a.sync_to(backend_b)
+ print("Getting updated diffs from Backend A to Backend B...")
+ backend_a.diff_to(backend_b).print_detailed()
if __name__ == "__main__":
diff --git a/examples/example1/models.py b/examples/example1/models.py
index c91b0139..80020b3f 100644
--- a/examples/example1/models.py
+++ b/examples/example1/models.py
@@ -25,8 +25,8 @@ class Device(DSyncModel):
_children = {"interface": "interfaces"}
name: str
- site_name: str
- role: str
+ site_name: Optional[str] # note that this attribute is NOT included in _attributes
+ role: Optional[str] # note that this attribute is NOT included in _attributes
interfaces: List = list()
diff --git a/poetry.lock b/poetry.lock
index 6129af59..f1df2fc1 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -122,9 +122,8 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "7.1.2"
[[package]]
-category = "dev"
+category = "main"
description = "Cross-platform colored terminal text."
-marker = "sys_platform == \"win32\" or platform_system == \"Windows\""
name = "colorama"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
@@ -551,6 +550,18 @@ pytest = ">=4.6"
[package.extras]
testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"]
+[[package]]
+category = "dev"
+description = "Structured logging assertions"
+name = "pytest-structlog"
+optional = false
+python-versions = "*"
+version = "0.3"
+
+[package.dependencies]
+pytest = "*"
+structlog = "*"
+
[[package]]
category = "dev"
description = "YAML parser and emitter for Python"
@@ -602,7 +613,7 @@ fixture = ["fixtures"]
test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"]
[[package]]
-category = "dev"
+category = "main"
description = "Python 2 and 3 compatibility utilities"
name = "six"
optional = false
@@ -640,6 +651,23 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0"
python = "<3.8"
version = ">=1.7.0"
+[[package]]
+category = "main"
+description = "Structured Logging for Python"
+name = "structlog"
+optional = false
+python-versions = "*"
+version = "20.1.0"
+
+[package.dependencies]
+six = "*"
+
+[package.extras]
+azure-pipelines = ["coverage", "freezegun (>=0.2.8)", "pretend", "pytest (>=3.3.0)", "simplejson", "pytest-azurepipelines", "python-rapidjson", "pytest-asyncio"]
+dev = ["coverage", "freezegun (>=0.2.8)", "pretend", "pytest (>=3.3.0)", "simplejson", "sphinx", "twisted", "pre-commit", "python-rapidjson", "pytest-asyncio"]
+docs = ["sphinx", "twisted"]
+tests = ["coverage", "freezegun (>=0.2.8)", "pretend", "pytest (>=3.3.0)", "simplejson", "python-rapidjson", "pytest-asyncio"]
+
[[package]]
category = "dev"
description = "Python Library for Tom's Obvious, Minimal Language"
@@ -733,7 +761,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["jaraco.itertools", "func-timeout"]
[metadata]
-content-hash = "bea8d554afbb65ce4e439f8b4c026abcbef2a308a587a6f5bd1ad4dbdb8da4c6"
+content-hash = "df11dba646e7bbc5b3c4379563d115085f6e12a0438db1397b17d0782e4bfe37"
python-versions = "^3.7"
[metadata.files]
@@ -1008,6 +1036,10 @@ pytest-cov = [
{file = "pytest-cov-2.10.1.tar.gz", hash = "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e"},
{file = "pytest_cov-2.10.1-py2.py3-none-any.whl", hash = "sha256:45ec2d5182f89a81fc3eb29e3d1ed3113b9e9a873bcddb2a71faaab066110191"},
]
+pytest-structlog = [
+ {file = "pytest-structlog-0.3.tar.gz", hash = "sha256:be01025a712f03e9e2ede3bfbd9bce7ce79d4b7449d426fc603bebfcb0b7e860"},
+ {file = "pytest_structlog-0.3-py2.py3-none-any.whl", hash = "sha256:c07ab9c89ee942ca5776ac81c746387dcb26d52c53466ee850daab67bb5b0281"},
+]
pyyaml = [
{file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"},
{file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"},
@@ -1067,6 +1099,10 @@ stevedore = [
{file = "stevedore-3.2.1-py3-none-any.whl", hash = "sha256:ddc09a744dc224c84ec8e8efcb70595042d21c97c76df60daee64c9ad53bc7ee"},
{file = "stevedore-3.2.1.tar.gz", hash = "sha256:a34086819e2c7a7f86d5635363632829dab8014e5fd7be2454c7cba84ac7514e"},
]
+structlog = [
+ {file = "structlog-20.1.0-py2.py3-none-any.whl", hash = "sha256:8a672be150547a93d90a7d74229a29e765be05bd156a35cdcc527ebf68e9af92"},
+ {file = "structlog-20.1.0.tar.gz", hash = "sha256:7a48375db6274ed1d0ae6123c486472aa1d0890b08d314d2b016f3aa7f35990b"},
+]
toml = [
{file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"},
{file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"},
diff --git a/pyproject.toml b/pyproject.toml
index 1c12d475..21eda1a2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,6 +7,8 @@ authors = ["Network to Code, LLC <info@networktocode.com>"]
[tool.poetry.dependencies]
python = "^3.7"
pydantic = "^1.6.1"
+structlog = "^20.1.0"
+colorama = {version = "^0.4.3", optional = true}
[tool.poetry.dev-dependencies]
pytest = "^6.1.0"
@@ -22,6 +24,7 @@ flake8 = "^3.8.3"
ipython = "^7.18.1"
mypy = "^0.782"
pytest-cov = "^2.10.1"
+pytest-structlog = "^0.3"
[tool.poetry.scripts]
dsync = 'dsync.cli:main'
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index 65e84c8e..17410373 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -287,6 +287,8 @@ class BackendB(GenericBackend):
device = DeviceB
place = PlaceB
+ type = "Backend_B"
+
DATA = {
"nyc": {
"nyc-spine1": {"role": "spine", "interfaces": {"eth0": "Interface 0/0", "eth1": "Interface 1"}},
@@ -313,7 +315,7 @@ def load(self):
@pytest.fixture
def backend_b():
"""Provide an instance of BackendB subclass of DSync."""
- dsync = BackendB()
+ dsync = BackendB(name="backend-b")
dsync.load()
return dsync
diff --git a/tests/unit/test_diff_element.py b/tests/unit/test_diff_element.py
index 85e9589c..7ccf6550 100644
--- a/tests/unit/test_diff_element.py
+++ b/tests/unit/test_diff_element.py
@@ -10,6 +10,8 @@ def test_diff_element_empty():
assert element.type == "interface"
assert element.name == "eth0"
assert element.keys == {"device_name": "device1", "name": "eth0"}
+ assert element.source_name == "source"
+ assert element.dest_name == "dest"
assert element.source_attrs is None
assert element.dest_attrs is None
assert list(element.get_children()) == []
@@ -18,6 +20,11 @@ def test_diff_element_empty():
assert not element.has_diffs(include_children=False)
assert element.get_attrs_keys() == []
+ element2 = DiffElement(
+ "interface", "eth0", {"device_name": "device1", "name": "eth0"}, source_name="S1", dest_name="D1"
+ )
+ assert element2.source_name == "S1"
+ assert element2.dest_name == "D1"
# TODO: test print_detailed
diff --git a/tests/unit/test_dsync.py b/tests/unit/test_dsync.py
index 4e0d10ef..4c7036ff 100644
--- a/tests/unit/test_dsync.py
+++ b/tests/unit/test_dsync.py
@@ -1,7 +1,5 @@
"""Unit tests for the DSync class."""
-import logging
-
import pytest
from dsync import DSync, DSyncModel, DSyncFlags
@@ -71,6 +69,8 @@ def test_generic_dsync_methods(generic_dsync, generic_dsync_model):
)
assert len(diff_elements) == 1
assert not diff_elements[0].has_diffs()
+ assert diff_elements[0].source_name == "DSync"
+ assert diff_elements[0].dest_name == "DSync"
def test_dsync_subclass_validation():
@@ -175,6 +175,19 @@ def test_dsync_subclass_methods_diff_sync(backend_a, backend_b):
assert backend_a.get_by_uids(["nyc", "sfo"], "device") == []
+def test_dsync_subclass_methods_name_type(backend_a, backend_b):
+ """Test DSync name and type an actual concrete subclass.
+
+ backend_a is using the default name and type
+ backend_b is using a user defined name and type
+ """
+ assert backend_a.name == "BackendA"
+ assert backend_a.type == "BackendA"
+
+ assert backend_b.name == "backend-b"
+ assert backend_b.type == "Backend_B"
+
+
def test_dsync_subclass_methods_crud(backend_a):
"""Test DSync CRUD APIs against a concrete subclass."""
site_nyc_a = backend_a.get(Site, "nyc")
@@ -199,9 +212,8 @@ def test_dsync_subclass_methods_crud(backend_a):
backend_a.remove(site_atl_a)
-def test_dsync_subclass_methods_sync_exceptions(caplog, error_prone_backend_a, backend_b):
+def test_dsync_subclass_methods_sync_exceptions(log, error_prone_backend_a, backend_b):
"""Test handling of exceptions during a sync."""
- caplog.set_level(logging.INFO)
with pytest.raises(ObjectCrudException):
error_prone_backend_a.sync_from(backend_b)
@@ -211,9 +223,15 @@ def test_dsync_subclass_methods_sync_exceptions(caplog, error_prone_backend_a, b
remaining_diffs.print_detailed()
assert remaining_diffs.has_diffs()
+ # At least some operations of each type should have succeeded
+ assert log.has("Created successfully", status="success")
+ assert log.has("Updated successfully", status="success")
+ assert log.has("Deleted successfully", status="success")
# Some ERROR messages should have been logged
- assert [record.message for record in caplog.records if record.levelname == "ERROR"] != []
- caplog.clear()
+ assert [event for event in log.events if event["level"] == "error"] != []
+ # Some messages with status="error" should have been logged - these may be the same as the above
+ assert [event for event in log.events if event.get("status") == "error"] != []
+ log.events = []
# Retry up to 10 times, we should sync successfully eventually
for i in range(10):
@@ -223,11 +241,15 @@ def test_dsync_subclass_methods_sync_exceptions(caplog, error_prone_backend_a, b
remaining_diffs.print_detailed()
if remaining_diffs.has_diffs():
# If we still have diffs, some ERROR messages should have been logged
- assert [record.message for record in caplog.records if record.levelname == "ERROR"] != []
- caplog.clear()
+ assert [event for event in log.events if event["level"] == "error"] != []
+ # Some messages with status="errored" should have been logged - these may be the same as the above
+ assert [event for event in log.events if event.get("status") == "error"] != []
+ log.events = []
else:
# No error messages should have been logged on the last, fully successful attempt
- assert [record.message for record in caplog.records if record.levelname == "ERROR"] == []
+ assert [event for event in log.events if event["level"] == "error"] == []
+ # Something must have succeeded for us to be done
+ assert [event for event in log.events if event.get("status") == "success"] != []
break
else:
pytest.fail("Sync was still incomplete after 10 retries")