Getting tests running locally and fixing bugs found in tests

datadog_sync/model/authn_mappings.py

@@ -19,6 +19,11 @@ class AuthNMappings(BaseResource):
             "attributes.saml_assertion_attribute_id",
             "relationships.saml_assertion_attribute",
         ],
+        non_nullable_attr=["relationships.team", "relationships.role"],
+        null_values={
+            "team": [{"data": None}],
+            "role": [{"data": None}],
+        },
         resource_connections={"roles": ["relationships.role.data.id"], "teams": ["relationships.team.data.id"]},
     )
     # Additional AuthNMappings specific attributes
@@ -49,6 +54,7 @@ async def pre_apply_hook(self) -> None:
     async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
         destination_client = self.config.destination_client
         payload = {"data": resource}
+        self.remove_null_relationships(payload)
         resp = await destination_client.post(self.resource_config.base_path, payload)
         self.remove_null_relationships(resp)
 
@@ -59,6 +65,7 @@ async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
         d_id = self.config.state.destination[self.resource_type][_id]["id"]
         resource["id"] = d_id
         payload = {"data": resource}
+        self.remove_null_relationships(payload)
         resp = await destination_client.patch(self.resource_config.base_path + f"/{d_id}", payload)
         self.remove_null_relationships(resp)
 

datadog_sync/model/logs_pipelines.py

@@ -85,6 +85,7 @@ async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
                 subdomain = f"{self.logs_intake_subdomain}.{destination_client.url_object.subdomain}"
 
             await destination_client.post(self.logs_intake_path, payload, subdomain=subdomain)
+
             created = False
             for _ in range(12):
                 updated_pipelines = await self.get_destination_integration_pipelines()

datadog_sync/model/monitors.py

@@ -22,6 +22,7 @@ class Monitors(BaseResource):
             "monitors": ["query"],
             "roles": ["restricted_roles"],
             "service_level_objectives": ["query"],
+            "restriction_policies": ["restriction_policy"],
         },
         base_path="/api/v1/monitor",
         excluded_attributes=[
@@ -36,6 +37,7 @@ class Monitors(BaseResource):
             "overall_state",
             "overall_state_modified",
         ],
+        non_nullable_attr=["restriction_policy"],
         tagging_config=TaggingConfig(path="tags"),
     )
     # Additional Monitors specific attributes

datadog_sync/model/notebooks.py

@@ -24,6 +24,10 @@ class Notebooks(BaseResource):
             "attributes.author",
             "attributes.metadata",
         ],
+        non_nullable_attr=["attributes.schema_version"],
+        null_values={
+            "schema_version": [0],
+        },
     )
     # Additional Notebooks specific attributes
     pagination_config = PaginationConfig(

datadog_sync/model/roles.py

@@ -18,7 +18,15 @@ class Roles(BaseResource):
     resource_type = "roles"
     resource_config = ResourceConfig(
         base_path="/api/v2/roles",
-        excluded_attributes=["id", "attributes.created_at", "attributes.modified_at", "attributes.user_count"],
+        excluded_attributes=[
+            "attributes.created_at",
+            "attributes.created_by_handle",
+            "attributes.managed",
+            "attributes.modified_at",
+            "attributes.modified_by_handle",
+            "attributes.user_count",
+            "id",
+        ],
     )
     # Additional Roles specific attributes
     source_permissions: Dict = {}
@@ -66,21 +74,36 @@ async def pre_resource_action_hook(self, _id, resource: Dict) -> None:
         await self.remap_permissions(resource)
 
     async def create_resource(self, _id, resource) -> Tuple[str, Dict]:
-        if resource["attributes"]["name"] in self.destination_roles_mapping:
-            role_copy = copy.deepcopy(resource)
-            role_copy.update(self.destination_roles_mapping[resource["attributes"]["name"]])
-
-            if check_diff(self.resource_config, resource, role_copy):
-                self.config.state.destination[self.resource_type][_id] = role_copy
-                return await self.update_resource(_id, resource)
-            else:
-                return _id, role_copy
-
-        destination_client = self.config.destination_client
-        payload = {"data": resource}
-        resp = await destination_client.post(self.resource_config.base_path, payload)
-
-        return _id, resp["data"]
+        # this method uses role name from matching
+        role_name = resource["attributes"]["name"]
+
+        # remove the 'managed' attribute since it can not be passed into creation
+        resource["attributes"].pop("managed", None)
+
+        # role does not exist at the destination, so create it
+        if role_name not in self.destination_roles_mapping:
+            destination_client = self.config.destination_client
+            payload = {"data": resource}
+            resp = await destination_client.post(self.resource_config.base_path, payload)
+            return _id, resp["data"]
+
+        # role already exists at the destination
+        matching_destination_role = self.destination_roles_mapping[role_name]
+        role_copy = copy.deepcopy(resource)
+        role_copy.update(matching_destination_role)
+
+        # role is managed at the destination, do nothing
+        if "managed" in matching_destination_role["attributes"] and matching_destination_role["attributes"]["managed"]:
+            self.config.logger.warning(f"{role_name} is a managed resource at the destination, can not update it")
+            return _id, role_copy
+
+        # role is not managed at destination and it differs
+        if check_diff(self.resource_config, resource, role_copy):
+            self.config.state.destination[self.resource_type][_id] = role_copy
+            return await self.update_resource(_id, resource)
+
+        # role is not managed at destination and does not differ
+        return _id, role_copy
 
     async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
         destination_client = self.config.destination_client

datadog_sync/model/synthetics_tests.py

@@ -39,6 +39,18 @@ class SyntheticsTests(BaseResource):
             "overall_state_modified",
             "stepCount",
         ],
+        non_nullable_attr=[
+            "options.monitor_options.on_missing_data",
+            "options.monitor_options.notify_audit",
+            "options.monitor_options.new_host_delay",
+            "options.monitor_options.include_tags",
+        ],
+        null_values={
+            "on_missing_data": ["show_no_data"],
+            "notify_audit": [False],
+            "new_host_delay": [300],
+            "include_tags": [True],
+        },
         tagging_config=TaggingConfig(path="tags"),
     )
     # Additional SyntheticsTests specific attributes

datadog_sync/utils/base_resource.py

@@ -55,6 +55,7 @@ class ResourceConfig:
     base_path: str
     resource_connections: Optional[Dict[str, List[str]]] = None
     non_nullable_attr: Optional[List[str]] = None
+    null_values: Optional[Dict[str]] = None
     excluded_attributes: Optional[List[str]] = None
     concurrent: bool = True
     deep_diff_config: dict = field(default_factory=lambda: {"ignore_order": True})

datadog_sync/utils/resource_utils.py

@@ -175,7 +175,7 @@ def remove_non_nullable_attributes(resource_config, resource):
     if resource_config.non_nullable_attr:
         for key in resource_config.non_nullable_attr:
             k_list = key.split(".")
-            del_null_attr(k_list, resource)
+            del_null_attr(resource_config, k_list, resource)
 
 
 def del_attr(k_list, resource):
@@ -187,11 +187,16 @@ def del_attr(k_list, resource):
         del_attr(k_list[1:], resource[k_list[0]])
 
 
-def del_null_attr(k_list, resource):
+def del_null_attr(resource_config, k_list, resource):
+    # the nulls get converted to "something", this converts them back
+    if resource_config.null_values and len(k_list) == 1 and k_list[0] in resource_config.null_values:
+        if k_list[0] in resource and resource[k_list[0]] in resource_config.null_values[k_list[0]]:
+            resource[k_list[0]] = None
+
     if len(k_list) == 1 and k_list[0] in resource and resource[k_list[0]] is None:
         resource.pop(k_list[0], None)
     elif len(k_list) > 1 and resource[k_list[0]] is not None:
-        del_null_attr(k_list[1:], resource[k_list[0]])
+        del_null_attr(resource_config, k_list[1:], resource[k_list[0]])
 
 
 def check_diff(resource_config, resource, state):

datadog_sync/utils/resources_handler.py

@@ -53,6 +53,8 @@ async def apply_resources(self) -> Tuple[int, int]:
                 self.worker.work_queue.put_nowait(item)
             await self.worker.schedule_workers()
             self.config.logger.info("finished importing missing dependencies")
+        else:
+            self.config.logger.info("did not import missing dependencies...")
 
         # handle resource cleanups
         if self.config.cleanup != FALSE:
@@ -194,7 +196,12 @@ async def _diffs_worker_cb(self, q_item: List) -> None:
                 return
 
             if _id in self.config.state.destination[resource_type]:
-                diff = check_diff(r_class.resource_config, self.config.state.destination[resource_type][_id], resource)
+                # We have to compare the prepared versions to deal w/ non-nullable attributes
+                destination_copy = deepcopy(self.config.state.destination[resource_type][_id])
+                resource_copy = deepcopy(resource)
+                prep_resource(r_class.resource_config, destination_copy)
+                prep_resource(r_class.resource_config, resource_copy)
+                diff = check_diff(r_class.resource_config, destination_copy, resource_copy)
                 if diff:
                     self.config.logger.info("diff: \n {}".format(pformat(diff)), resource_type=resource_type, _id=_id)
             else:

tests/integration/cassettes/test_cli/TestCli.test_cleanup.frozen

@@ -1 +1 @@
-2024-02-02T21:32:02.256215-05:00
+2024-11-01T10:45:06.954992-04:00